diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 917dfa28da7..c5db1438d8b 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -65,6 +65,9 @@ New Features Polygon instances from a standard GeoJSON string (Robert Muir, Mike McCandless) +* SOLR-9279: Queries module: new ComparisonBoolFunction base class + (Doug Turnbull via David Smiley) + Bug Fixes * LUCENE-6662: Fixed potential resource leaks. (Rishabh Patel via Adrien Grand) @@ -135,10 +138,9 @@ Improvements * LUCENE-7385: Improve/fix assert messages in SpanScorer. (David Smiley) -* LUCENE-7390: Improve performance of indexing points by allowing the - codec to use transient heap in proportion to IndexWriter's RAM - buffer, instead of a fixed 16.0 MB. A custom codec can still - override the buffer size itself. (Mike McCandless) +* LUCENE-7393: Add ICUTokenizer option to parse Myanmar text as syllables instead of words, + because the ICU word-breaking algorithm has some issues. This allows for the previous + tokenization used before Lucene 5. (AM, Robert Muir) Optimizations @@ -154,6 +156,12 @@ Optimizations * LUCENE-7311: Cached term queries do not seek the terms dictionary anymore. (Adrien Grand) +* LUCENE-7396, LUCENE-7399: Faster flush of points. + (Adrien Grand, Mike McCandless) + +* LUCENE-7406: Automaton and PrefixQuery tweaks (fewer object (re)allocations). + (Christine Poerschke) + Other * LUCENE-4787: Fixed some highlighting javadocs. (Michael Dodsworth via Adrien diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/minhash/MinHashFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/minhash/MinHashFilter.java index 1a1a637d0a3..61c816d5640 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/minhash/MinHashFilter.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/minhash/MinHashFilter.java @@ -402,6 +402,7 @@ public class MinHashFilter extends TokenFilter { } /** Returns the MurmurHash3_x64_128 hash, placing the result in "out". */ + @SuppressWarnings("fallthrough") // the huge switch is designed to use fall through into cases! static void murmurhash3_x64_128(byte[] key, int offset, int len, int seed, LongPair out) { // The original algorithm does have a 32 bit unsigned seed. // We have to mask to match the behavior of the unsigned types and prevent sign extension. diff --git a/lucene/analysis/icu/src/data/uax29/MyanmarSyllable.rbbi b/lucene/analysis/icu/src/data/uax29/MyanmarSyllable.rbbi new file mode 100644 index 00000000000..18408037dca --- /dev/null +++ b/lucene/analysis/icu/src/data/uax29/MyanmarSyllable.rbbi @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# Parses Myanmar text, with syllable as token. +# + +$Cons = [[:Other_Letter:]&[:Myanmar:]]; +$Virama = [\u1039]; +$Asat = [\u103A]; + +$WordJoin = [:Line_Break=Word_Joiner:]; + +# +# default numerical definitions +# +$Extend = [\p{Word_Break = Extend}]; +$Format = [\p{Word_Break = Format}]; +$MidNumLet = [\p{Word_Break = MidNumLet}]; +$MidNum = [\p{Word_Break = MidNum}]; +$Numeric = [\p{Word_Break = Numeric}]; +$ExtendNumLet = [\p{Word_Break = ExtendNumLet}]; +$MidNumLetEx = $MidNumLet ($Extend | $Format)*; +$MidNumEx = $MidNum ($Extend | $Format)*; +$NumericEx = $Numeric ($Extend | $Format)*; +$ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*; + +$ConsEx = $Cons ($Extend | $Format)*; +$AsatEx = $Cons $Asat ($Virama $ConsEx)? ($Extend | $Format)*; +$MyanmarSyllableEx = $ConsEx ($Virama $ConsEx)? ($AsatEx)*; +$MyanmarJoinedSyllableEx = $MyanmarSyllableEx ($WordJoin $MyanmarSyllableEx)*; + +!!forward; +$MyanmarJoinedSyllableEx {200}; + +# default numeric rules +$NumericEx $ExtendNumLetEx? (($MidNumEx | $MidNumLetEx)? $NumericEx $ExtendNumLetEx?)* {100}; diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/DefaultICUTokenizerConfig.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/DefaultICUTokenizerConfig.java index b33663d1e2d..3cd62c8b2fc 100644 --- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/DefaultICUTokenizerConfig.java +++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/DefaultICUTokenizerConfig.java @@ -63,9 +63,12 @@ public class DefaultICUTokenizerConfig extends ICUTokenizerConfig { // the same as ROOT, except no dictionary segmentation for cjk private static final BreakIterator defaultBreakIterator = readBreakIterator("Default.brk"); + private static final BreakIterator myanmarSyllableIterator = + readBreakIterator("MyanmarSyllable.brk"); // TODO: deprecate this boolean? you only care if you are doing super-expert stuff... private final boolean cjkAsWords; + private final boolean myanmarAsWords; /** * Creates a new config. This object is lightweight, but the first @@ -74,9 +77,12 @@ public class DefaultICUTokenizerConfig extends ICUTokenizerConfig { * otherwise text will be segmented according to UAX#29 defaults. * If this is true, all Han+Hiragana+Katakana words will be tagged as * IDEOGRAPHIC. + * @param myanmarAsWords true if Myanmar text should undergo dictionary-based segmentation, + * otherwise it will be tokenized as syllables. */ - public DefaultICUTokenizerConfig(boolean cjkAsWords) { + public DefaultICUTokenizerConfig(boolean cjkAsWords, boolean myanmarAsWords) { this.cjkAsWords = cjkAsWords; + this.myanmarAsWords = myanmarAsWords; } @Override @@ -88,6 +94,12 @@ public class DefaultICUTokenizerConfig extends ICUTokenizerConfig { public BreakIterator getBreakIterator(int script) { switch(script) { case UScript.JAPANESE: return (BreakIterator)cjkBreakIterator.clone(); + case UScript.MYANMAR: + if (myanmarAsWords) { + return (BreakIterator)defaultBreakIterator.clone(); + } else { + return (BreakIterator)myanmarSyllableIterator.clone(); + } default: return (BreakIterator)defaultBreakIterator.clone(); } } diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java index 64c6785fb0e..09415516479 100644 --- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java +++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java @@ -68,7 +68,7 @@ public final class ICUTokenizer extends Tokenizer { * @see DefaultICUTokenizerConfig */ public ICUTokenizer() { - this(new DefaultICUTokenizerConfig(true)); + this(new DefaultICUTokenizerConfig(true, true)); } /** diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java index deb5d4fbbca..974e719880e 100644 --- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java +++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java @@ -79,6 +79,7 @@ public class ICUTokenizerFactory extends TokenizerFactory implements ResourceLoa private final Map tailored; private ICUTokenizerConfig config; private final boolean cjkAsWords; + private final boolean myanmarAsWords; /** Creates a new ICUTokenizerFactory */ public ICUTokenizerFactory(Map args) { @@ -95,6 +96,7 @@ public class ICUTokenizerFactory extends TokenizerFactory implements ResourceLoa } } cjkAsWords = getBoolean(args, "cjkAsWords", true); + myanmarAsWords = getBoolean(args, "myanmarAsWords", true); if (!args.isEmpty()) { throw new IllegalArgumentException("Unknown parameters: " + args); } @@ -104,7 +106,7 @@ public class ICUTokenizerFactory extends TokenizerFactory implements ResourceLoa public void inform(ResourceLoader loader) throws IOException { assert tailored != null : "init must be called first!"; if (tailored.isEmpty()) { - config = new DefaultICUTokenizerConfig(cjkAsWords); + config = new DefaultICUTokenizerConfig(cjkAsWords, myanmarAsWords); } else { final BreakIterator breakers[] = new BreakIterator[UScript.CODE_LIMIT]; for (Map.Entry entry : tailored.entrySet()) { @@ -112,7 +114,7 @@ public class ICUTokenizerFactory extends TokenizerFactory implements ResourceLoa String resourcePath = entry.getValue(); breakers[code] = parseRules(resourcePath, loader); } - config = new DefaultICUTokenizerConfig(cjkAsWords) { + config = new DefaultICUTokenizerConfig(cjkAsWords, myanmarAsWords) { @Override public BreakIterator getBreakIterator(int script) { diff --git a/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk new file mode 100644 index 00000000000..41b977b259a Binary files /dev/null and b/lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/MyanmarSyllable.brk differ diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java index 6398b2c0e94..027baa35705 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java @@ -42,7 +42,7 @@ public class TestICUTokenizer extends BaseTokenStreamTestCase { sb.append(whitespace); sb.append("testing 1234"); String input = sb.toString(); - ICUTokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false)); + ICUTokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true)); tokenizer.setReader(new StringReader(input)); assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" }); } @@ -53,7 +53,7 @@ public class TestICUTokenizer extends BaseTokenStreamTestCase { sb.append('a'); } String input = sb.toString(); - ICUTokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false)); + ICUTokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true)); tokenizer.setReader(new StringReader(input)); char token[] = new char[4096]; Arrays.fill(token, 'a'); @@ -75,7 +75,7 @@ public class TestICUTokenizer extends BaseTokenStreamTestCase { a = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false)); + Tokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true)); TokenFilter filter = new ICUNormalizer2Filter(tokenizer); return new TokenStreamComponents(tokenizer, filter); } diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java index a29686c68eb..96f44d686b0 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java @@ -34,7 +34,7 @@ public class TestICUTokenizerCJK extends BaseTokenStreamTestCase { a = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName) { - return new TokenStreamComponents(new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(true))); + return new TokenStreamComponents(new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(true, true))); } }; } diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestMyanmarSyllable.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestMyanmarSyllable.java new file mode 100644 index 00000000000..a3b608ed270 --- /dev/null +++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestMyanmarSyllable.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.analysis.icu.segmentation; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.Tokenizer; + +/** Test tokenizing Myanmar text into syllables */ +public class TestMyanmarSyllable extends BaseTokenStreamTestCase { + + Analyzer a; + + @Override + public void setUp() throws Exception { + super.setUp(); + a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, false)); + return new TokenStreamComponents(tokenizer); + } + }; + } + + @Override + public void tearDown() throws Exception { + a.close(); + super.tearDown(); + } + + /** as opposed to dictionary break of သက်ဝင်|လှုပ်ရှား|စေ|ပြီး */ + public void testBasics() throws Exception { + assertAnalyzesTo(a, "သက်ဝင်လှုပ်ရှားစေပြီး", new String[] { "သက်", "ဝင်", "လှုပ်", "ရှား", "စေ", "ပြီး" }); + } + + // simple tests from "A Rule-based Syllable Segmentation of Myanmar Text" + // * http://www.aclweb.org/anthology/I08-3010 + // (see also the presentation: http://gii2.nagaokaut.ac.jp/gii/media/share/20080901-ZMM%20Presentation.pdf) + // The words are fake, we just test the categories. + // note that currently our algorithm is not sophisticated enough to handle some of the special cases! + + /** constant */ + public void testC() throws Exception { + assertAnalyzesTo(a, "ကက", new String[] { "က", "က" }); + } + + /** consonant + sign */ + public void testCF() throws Exception { + assertAnalyzesTo(a, "ကံကံ", new String[] { "ကံ", "ကံ" }); + } + + /** consonant + consonant + asat */ + public void testCCA() throws Exception { + assertAnalyzesTo(a, "ကင်ကင်", new String[] { "ကင်", "ကင်" }); + } + + /** consonant + consonant + asat + sign */ + public void testCCAF() throws Exception { + assertAnalyzesTo(a, "ကင်းကင်း", new String[] { "ကင်း", "ကင်း" }); + } + + /** consonant + vowel */ + public void testCV() throws Exception { + assertAnalyzesTo(a, "ကာကာ", new String[] { "ကာ", "ကာ" }); + } + + /** consonant + vowel + sign */ + public void testCVF() throws Exception { + assertAnalyzesTo(a, "ကားကား", new String[] { "ကား", "ကား" }); + } + + /** consonant + vowel + vowel + asat */ + public void testCVVA() throws Exception { + assertAnalyzesTo(a, "ကော်ကော်", new String[] { "ကော်", "ကော်" }); + } + + /** consonant + vowel + vowel + consonant + asat */ + public void testCVVCA() throws Exception { + assertAnalyzesTo(a, "ကောင်ကောင်", new String[] { "ကောင်", "ကောင်" }); + } + + /** consonant + vowel + vowel + consonant + asat + sign */ + public void testCVVCAF() throws Exception { + assertAnalyzesTo(a, "ကောင်းကောင်း", new String[] { "ကောင်း", "ကောင်း" }); + } + + /** consonant + medial */ + public void testCM() throws Exception { + assertAnalyzesTo(a, "ကျကျ", new String[] { "ကျ", "ကျ" }); + } + + /** consonant + medial + sign */ + public void testCMF() throws Exception { + assertAnalyzesTo(a, "ကျံကျံ", new String[] { "ကျံ", "ကျံ" }); + } + + /** consonant + medial + consonant + asat */ + public void testCMCA() throws Exception { + assertAnalyzesTo(a, "ကျင်ကျင်", new String[] { "ကျင်", "ကျင်" }); + } + + /** consonant + medial + consonant + asat + sign */ + public void testCMCAF() throws Exception { + assertAnalyzesTo(a, "ကျင်းကျင်း", new String[] { "ကျင်း", "ကျင်း" }); + } + + /** consonant + medial + vowel */ + public void testCMV() throws Exception { + assertAnalyzesTo(a, "ကျာကျာ", new String[] { "ကျာ", "ကျာ" }); + } + + /** consonant + medial + vowel + sign */ + public void testCMVF() throws Exception { + assertAnalyzesTo(a, "ကျားကျား", new String[] { "ကျား", "ကျား" }); + } + + /** consonant + medial + vowel + vowel + asat */ + public void testCMVVA() throws Exception { + assertAnalyzesTo(a, "ကျော်ကျော်", new String[] { "ကျော်", "ကျော်" }); + } + + /** consonant + medial + vowel + vowel + consonant + asat */ + public void testCMVVCA() throws Exception { + assertAnalyzesTo(a, "ကြောင်ကြောင်", new String[] { "ကြောင်", "ကြောင်"}); + } + + /** consonant + medial + vowel + vowel + consonant + asat + sign */ + public void testCMVVCAF() throws Exception { + assertAnalyzesTo(a, "ကြောင်းကြောင်း", new String[] { "ကြောင်း", "ကြောင်း"}); + } + + /** independent vowel */ + public void testI() throws Exception { + assertAnalyzesTo(a, "ဪဪ", new String[] { "ဪ", "ဪ" }); + } + + /** independent vowel */ + public void testE() throws Exception { + assertAnalyzesTo(a, "ဣဣ", new String[] { "ဣ", "ဣ" }); + } +} diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java index 17ea9674f5f..411b85efafd 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java @@ -46,7 +46,7 @@ public class TestWithCJKBigramFilter extends BaseTokenStreamTestCase { analyzer = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false)); + Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true)); TokenStream result = new CJKBigramFilter(source); return new TokenStreamComponents(source, new StopFilter(result, CharArraySet.EMPTY_SET)); } @@ -60,7 +60,7 @@ public class TestWithCJKBigramFilter extends BaseTokenStreamTestCase { analyzer2 = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false)); + Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false, true)); // we put this before the CJKBigramFilter, because the normalization might combine // some halfwidth katakana forms, which will affect the bigramming. TokenStream result = new ICUNormalizer2Filter(source); diff --git a/lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java b/lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java index 374624b41c6..8bb0b1dcdc2 100644 --- a/lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java +++ b/lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java @@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.ScoreDoc; @@ -82,11 +83,20 @@ public class DatasetSplitter { // get the exact no. of existing classes int noOfClasses = 0; for (LeafReaderContext leave : originalIndex.leaves()) { + long valueCount = 0; SortedDocValues classValues = leave.reader().getSortedDocValues(classFieldName); - if (classValues == null) { - throw new IllegalStateException("the classFieldName \"" + classFieldName + "\" must index sorted doc values"); + if (classValues != null) { + valueCount = classValues.getValueCount(); + } else { + SortedSetDocValues sortedSetDocValues = leave.reader().getSortedSetDocValues(classFieldName); + if (sortedSetDocValues != null) { + valueCount = sortedSetDocValues.getValueCount(); + } } - noOfClasses += classValues.getValueCount(); + if (classValues == null) { + throw new IllegalStateException("field \"" + classFieldName + "\" must have sorted (set) doc values"); + } + noOfClasses += valueCount; } try { diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java index 67289b63c10..8d5c0344509 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointsWriter.java @@ -68,7 +68,7 @@ class SimpleTextPointsWriter extends PointsWriter { } @Override - public void writeField(FieldInfo fieldInfo, PointsReader values, double maxMBSortInHeap) throws IOException { + public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException { boolean singleValuePerDoc = values.size(fieldInfo.name) == values.getDocCount(fieldInfo.name); @@ -79,7 +79,7 @@ class SimpleTextPointsWriter extends PointsWriter { fieldInfo.getPointDimensionCount(), fieldInfo.getPointNumBytes(), BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE, - maxMBSortInHeap, + BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, values.size(fieldInfo.name), singleValuePerDoc) { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/MutablePointsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/MutablePointsReader.java new file mode 100644 index 00000000000..dccca26e012 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/codecs/MutablePointsReader.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs; + +import org.apache.lucene.util.BytesRef; + +/** {@link PointsReader} whose order of points can be changed. + * This class is useful for codecs to optimize flush. + * @lucene.internal */ +public abstract class MutablePointsReader extends PointsReader { + + /** Sole constructor. */ + protected MutablePointsReader() {} + + /** Set {@code packedValue} with a reference to the packed bytes of the i-th value. */ + public abstract void getValue(int i, BytesRef packedValue); + + /** Get the k-th byte of the i-th value. */ + public abstract byte getByteAt(int i, int k); + + /** Return the doc ID of the i-th value. */ + public abstract int getDocID(int i); + + /** Swap the i-th and j-th values. */ + public abstract void swap(int i, int j); + +} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java index 845849780a8..05084db6ca1 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java @@ -22,7 +22,6 @@ import java.io.IOException; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.MergeState; -import org.apache.lucene.util.bkd.BKDWriter; /** Abstract API to write points * @@ -35,9 +34,8 @@ public abstract class PointsWriter implements Closeable { protected PointsWriter() { } - /** Write all values contained in the provided reader. {@code maxMBSortInHeap} is the maximum - * transient heap that can be used to sort values, before spilling to disk for offline sorting */ - public abstract void writeField(FieldInfo fieldInfo, PointsReader values, double maxMBSortInHeap) throws IOException; + /** Write all values contained in the provided reader */ + public abstract void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException; /** Default naive merge implementation for one field: it just re-indexes all the values * from the incoming segment. The default codec overrides this for 1D fields and uses @@ -147,10 +145,7 @@ public abstract class PointsWriter implements Closeable { public int getDocCount(String fieldName) { return finalDocCount; } - }, - // TODO: also let merging of > 1D fields tap into IW's indexing buffer size, somehow (1D fields do an optimized merge sort - // and don't need heap) - BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP); + }); } /** Default merge implementation to merge incoming points readers by visiting all their points and diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java index 3acfac3f8b0..ff9de58091c 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointsWriter.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.MutablePointsReader; import org.apache.lucene.codecs.PointsReader; import org.apache.lucene.codecs.PointsWriter; import org.apache.lucene.index.FieldInfo; @@ -39,9 +40,7 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.bkd.BKDReader; import org.apache.lucene.util.bkd.BKDWriter; -/** Writes dimensional values - * - * @lucene.experimental */ +/** Writes dimensional values */ public class Lucene60PointsWriter extends PointsWriter implements Closeable { /** Output used to write the BKD tree data file */ @@ -52,13 +51,15 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable { final SegmentWriteState writeState; final int maxPointsInLeafNode; + final double maxMBSortInHeap; private boolean finished; /** Full constructor */ - public Lucene60PointsWriter(SegmentWriteState writeState, int maxPointsInLeafNode) throws IOException { + public Lucene60PointsWriter(SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap) throws IOException { assert writeState.fieldInfos.hasPointValues(); this.writeState = writeState; this.maxPointsInLeafNode = maxPointsInLeafNode; + this.maxMBSortInHeap = maxMBSortInHeap; String dataFileName = IndexFileNames.segmentFileName(writeState.segmentInfo.name, writeState.segmentSuffix, Lucene60PointsFormat.DATA_EXTENSION); @@ -80,11 +81,11 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable { /** Uses the defaults values for {@code maxPointsInLeafNode} (1024) and {@code maxMBSortInHeap} (16.0) */ public Lucene60PointsWriter(SegmentWriteState writeState) throws IOException { - this(writeState, BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE); + this(writeState, BKDWriter.DEFAULT_MAX_POINTS_IN_LEAF_NODE, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP); } @Override - public void writeField(FieldInfo fieldInfo, PointsReader values, double maxMBSortInHeap) throws IOException { + public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException { boolean singleValuePerDoc = values.size(fieldInfo.name) == values.getDocCount(fieldInfo.name); @@ -98,6 +99,14 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable { values.size(fieldInfo.name), singleValuePerDoc)) { + if (values instanceof MutablePointsReader) { + final long fp = writer.writeField(dataOut, fieldInfo.name, (MutablePointsReader) values); + if (fp != -1) { + indexFPs.put(fieldInfo.name, fp); + } + return; + } + values.intersect(fieldInfo.name, new IntersectVisitor() { @Override public void visit(int docID) { @@ -173,8 +182,7 @@ public class Lucene60PointsWriter extends PointsWriter implements Closeable { fieldInfo.getPointDimensionCount(), fieldInfo.getPointNumBytes(), maxPointsInLeafNode, - // NOTE: not used, since BKDWriter.merge does a merge sort: - BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP, + maxMBSortInHeap, totMaxSize, singleValuePerDoc)) { List bkdReaders = new ArrayList<>(); diff --git a/lucene/core/src/java/org/apache/lucene/document/Field.java b/lucene/core/src/java/org/apache/lucene/document/Field.java index 37fa255c972..87986101f3b 100644 --- a/lucene/core/src/java/org/apache/lucene/document/Field.java +++ b/lucene/core/src/java/org/apache/lucene/document/Field.java @@ -257,7 +257,7 @@ public class Field implements IndexableField { /** * The value of the field as a String, or null. If null, the Reader value or * binary value is used. Exactly one of stringValue(), readerValue(), and - * getBinaryValue() must be set. + * binaryValue() must be set. */ @Override public String stringValue() { @@ -271,7 +271,7 @@ public class Field implements IndexableField { /** * The value of the field as a Reader, or null. If null, the String value or * binary value is used. Exactly one of stringValue(), readerValue(), and - * getBinaryValue() must be set. + * binaryValue() must be set. */ @Override public Reader readerValue() { @@ -420,7 +420,7 @@ public class Field implements IndexableField { /** * Expert: sets the token stream to be used for indexing and causes * isIndexed() and isTokenized() to return true. May be combined with stored - * values from stringValue() or getBinaryValue() + * values from stringValue() or binaryValue() */ public void setTokenStream(TokenStream tokenStream) { if (type.indexOptions() == IndexOptions.NONE || !type.tokenized()) { diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index 351235e2b5d..e72145c93f6 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -153,7 +153,7 @@ class DocumentsWriterPerThread { final Allocator byteBlockAllocator; final IntBlockPool.Allocator intBlockAllocator; private final AtomicLong pendingNumDocs; - final LiveIndexWriterConfig indexWriterConfig; + private final LiveIndexWriterConfig indexWriterConfig; private final boolean enableTestPoints; private final IndexWriter indexWriter; diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index bdd95526429..0fb23d96b36 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -762,7 +762,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable { * {@link #getConfig()}. * *

- * NOTE: after this writer is created, the given configuration instance + * NOTE: after ths writer is created, the given configuration instance * cannot be passed to another writer. * * @param d diff --git a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java index fe4924df6c7..cec70c099aa 100644 --- a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java +++ b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java @@ -168,14 +168,9 @@ public class LiveIndexWriterConfig { /** * Determines the amount of RAM that may be used for buffering added documents - * and deletions before beginning to flush them to the Directory. For - * faster indexing performance it's best to use as large a RAM buffer as you can. - *

- * Note that this setting is not a hard limit on memory usage during indexing, as - * transient and non-trivial memory well beyond this buffer size may be used, - * for example due to segment merges or writing points to new segments. - * For application stability the available memory in the JVM - * should be significantly larger than the RAM buffer used for indexing. + * and deletions before they are flushed to the Directory. Generally for + * faster indexing performance it's best to flush by RAM usage instead of + * document count and use as large a RAM buffer as you can. *

* When this is set, the writer will flush whenever buffered documents and * deletions use this much RAM. Pass in @@ -183,6 +178,14 @@ public class LiveIndexWriterConfig { * due to RAM usage. Note that if flushing by document count is also enabled, * then the flush will be triggered by whichever comes first. *

+ * The maximum RAM limit is inherently determined by the JVMs available + * memory. Yet, an {@link IndexWriter} session can consume a significantly + * larger amount of memory than the given RAM limit since this limit is just + * an indicator when to flush memory resident documents to the Directory. + * Flushes are likely happen concurrently while other threads adding documents + * to the writer. For application stability the available memory in the JVM + * should be significantly larger than the RAM buffer used for indexing. + *

* NOTE: the account of RAM usage for pending deletions is only * approximate. Specifically, if you delete by Query, Lucene currently has no * way to measure the RAM usage of individual Queries so the accounting will diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java index 511635c14ed..daf1f338730 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java @@ -18,13 +18,13 @@ package org.apache.lucene.index; import java.io.IOException; +import org.apache.lucene.codecs.MutablePointsReader; import org.apache.lucene.codecs.PointsReader; import org.apache.lucene.codecs.PointsWriter; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; -import org.apache.lucene.util.bkd.BKDWriter; /** Buffers up pending byte[][] value(s) per doc, then flushes when segment flushes. */ class PointValuesWriter { @@ -35,8 +35,7 @@ class PointValuesWriter { private int numPoints; private int numDocs; private int lastDocID = -1; - private final byte[] packedValue; - private final LiveIndexWriterConfig indexWriterConfig; + private final int packedBytesLength; public PointValuesWriter(DocumentsWriterPerThread docWriter, FieldInfo fieldInfo) { this.fieldInfo = fieldInfo; @@ -44,8 +43,7 @@ class PointValuesWriter { this.bytes = new ByteBlockPool(docWriter.byteBlockAllocator); docIDs = new int[16]; iwBytesUsed.addAndGet(16 * Integer.BYTES); - packedValue = new byte[fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes()]; - indexWriterConfig = docWriter.indexWriterConfig; + packedBytesLength = fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes(); } // TODO: if exactly the same value is added to exactly the same doc, should we dedup? @@ -53,9 +51,10 @@ class PointValuesWriter { if (value == null) { throw new IllegalArgumentException("field=" + fieldInfo.name + ": point value must not be null"); } - if (value.length != fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes()) { + if (value.length != packedBytesLength) { throw new IllegalArgumentException("field=" + fieldInfo.name + ": this field's value has length=" + value.length + " but should be " + (fieldInfo.getPointDimensionCount() * fieldInfo.getPointNumBytes())); } + if (docIDs.length == numPoints) { docIDs = ArrayUtil.grow(docIDs, numPoints+1); iwBytesUsed.addAndGet((docIDs.length - numPoints) * Integer.BYTES); @@ -66,68 +65,111 @@ class PointValuesWriter { numDocs++; lastDocID = docID; } + numPoints++; } public void flush(SegmentWriteState state, PointsWriter writer) throws IOException { + PointsReader reader = new MutablePointsReader() { - writer.writeField(fieldInfo, - new PointsReader() { - @Override - public void intersect(String fieldName, IntersectVisitor visitor) throws IOException { - if (fieldName.equals(fieldInfo.name) == false) { - throw new IllegalArgumentException("fieldName must be the same"); - } - for(int i=0;i void select(T[] arr, int from, int to, int k, Comparator comparator) { - if (k < from) { - throw new IllegalArgumentException("k must be >= from"); - } - if (k >= to) { - throw new IllegalArgumentException("k must be < to"); - } - final int maxDepth = 2 * MathUtil.log(to - from, 2); - quickSelect(arr, from, to, k, comparator, maxDepth); + public static void select(T[] arr, int from, int to, int k, Comparator comparator) { + new IntroSelector() { + + T pivot; + + @Override + protected void swap(int i, int j) { + ArrayUtil.swap(arr, i, j); + } + + @Override + protected void setPivot(int i) { + pivot = arr[i]; + } + + @Override + protected int comparePivot(int j) { + return comparator.compare(pivot, arr[j]); + } + }.select(from, to, k); } - private static void quickSelect(T[] arr, int from, int to, int k, Comparator comparator, int maxDepth) { - assert from <= k; - assert k < to; - if (to - from == 1) { - return; - } - if (--maxDepth < 0) { - Arrays.sort(arr, from, to, comparator); - return; - } - - final int mid = (from + to) >>> 1; - // heuristic: we use the median of the values at from, to-1 and mid as a pivot - if (comparator.compare(arr[from], arr[to - 1]) > 0) { - swap(arr, from, to - 1); - } - if (comparator.compare(arr[to - 1], arr[mid]) > 0) { - swap(arr, to - 1, mid); - if (comparator.compare(arr[from], arr[to - 1]) > 0) { - swap(arr, from, to - 1); - } - } - - T pivot = arr[to - 1]; - - int left = from + 1; - int right = to - 2; - - for (;;) { - while (comparator.compare(pivot, arr[left]) > 0) { - ++left; - } - - while (left < right && comparator.compare(pivot, arr[right]) <= 0) { - --right; - } - - if (left < right) { - swap(arr, left, right); - --right; - } else { - break; - } - } - swap(arr, left, to - 1); - - if (left == k) { - return; - } else if (left < k) { - quickSelect(arr, left + 1, to, k, comparator, maxDepth); - } else { - quickSelect(arr, from, left, k, comparator, maxDepth); - } - } } diff --git a/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java b/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java index 6bb12bdfea6..e575d87317f 100644 --- a/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java +++ b/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java @@ -378,5 +378,34 @@ public final class ByteBlockPool { } } while (true); } + + /** + * Set the given {@link BytesRef} so that its content is equal to the + * {@code ref.length} bytes starting at {@code offset}. Most of the time this + * method will set pointers to internal data-structures. However, in case a + * value crosses a boundary, a fresh copy will be returned. + * On the contrary to {@link #setBytesRef(BytesRef, int)}, this does not + * expect the length to be encoded with the data. + */ + public void setRawBytesRef(BytesRef ref, final long offset) { + int bufferIndex = (int) (offset >> BYTE_BLOCK_SHIFT); + int pos = (int) (offset & BYTE_BLOCK_MASK); + if (pos + ref.length <= BYTE_BLOCK_SIZE) { + ref.bytes = buffers[bufferIndex]; + ref.offset = pos; + } else { + ref.bytes = new byte[ref.length]; + ref.offset = 0; + readBytes(offset, ref.bytes, 0, ref.length); + } + } + + /** Read a single byte at the given {@code offset}. */ + public byte readByte(long offset) { + int bufferIndex = (int) (offset >> BYTE_BLOCK_SHIFT); + int pos = (int) (offset & BYTE_BLOCK_MASK); + byte[] buffer = buffers[bufferIndex]; + return buffer[pos]; + } } diff --git a/lucene/core/src/java/org/apache/lucene/util/InPlaceMergeSorter.java b/lucene/core/src/java/org/apache/lucene/util/InPlaceMergeSorter.java index 1afba99f670..de8f97624bb 100644 --- a/lucene/core/src/java/org/apache/lucene/util/InPlaceMergeSorter.java +++ b/lucene/core/src/java/org/apache/lucene/util/InPlaceMergeSorter.java @@ -33,8 +33,8 @@ public abstract class InPlaceMergeSorter extends Sorter { } void mergeSort(int from, int to) { - if (to - from < INSERTION_SORT_THRESHOLD) { - insertionSort(from, to); + if (to - from < BINARY_SORT_THRESHOLD) { + binarySort(from, to); } else { final int mid = (from + to) >>> 1; mergeSort(from, mid); diff --git a/lucene/core/src/java/org/apache/lucene/util/IntroSelector.java b/lucene/core/src/java/org/apache/lucene/util/IntroSelector.java new file mode 100644 index 00000000000..298480182b2 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/IntroSelector.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.util; + +import java.util.Comparator; + +/** Implementation of the quick select algorithm. + *

It uses the median of the first, middle and last values as a pivot and + * falls back to a heap sort when the number of recursion levels exceeds + * {@code 2 lg(n)}, as a consequence it runs in linear time on average and in + * {@code n log(n)} time in the worst case.

+ * @lucene.internal */ +public abstract class IntroSelector extends Selector { + + @Override + public final void select(int from, int to, int k) { + checkArgs(from, to, k); + final int maxDepth = 2 * MathUtil.log(to - from, 2); + quickSelect(from, to, k, maxDepth); + } + + // heap sort + // TODO: use median of median instead to have linear worst-case rather than + // n*log(n) + void slowSelect(int from, int to, int k) { + new Sorter() { + + @Override + protected void swap(int i, int j) { + IntroSelector.this.swap(i, j); + } + + @Override + protected int compare(int i, int j) { + return IntroSelector.this.compare(i, j); + } + + public void sort(int from, int to) { + heapSort(from, to); + } + }.sort(from, to); + } + + private void quickSelect(int from, int to, int k, int maxDepth) { + assert from <= k; + assert k < to; + if (to - from == 1) { + return; + } + if (--maxDepth < 0) { + slowSelect(from, to, k); + return; + } + + final int mid = (from + to) >>> 1; + // heuristic: we use the median of the values at from, to-1 and mid as a pivot + if (compare(from, to - 1) > 0) { + swap(from, to - 1); + } + if (compare(to - 1, mid) > 0) { + swap(to - 1, mid); + if (compare(from, to - 1) > 0) { + swap(from, to - 1); + } + } + + setPivot(to - 1); + + int left = from + 1; + int right = to - 2; + + for (;;) { + while (comparePivot(left) > 0) { + ++left; + } + + while (left < right && comparePivot(right) <= 0) { + --right; + } + + if (left < right) { + swap(left, right); + --right; + } else { + break; + } + } + swap(left, to - 1); + + if (left == k) { + return; + } else if (left < k) { + quickSelect(left + 1, to, k, maxDepth); + } else { + quickSelect(from, left, k, maxDepth); + } + } + + /** Compare entries found in slots i and j. + * The contract for the returned value is the same as + * {@link Comparator#compare(Object, Object)}. */ + protected int compare(int i, int j) { + setPivot(i); + return comparePivot(j); + } + + /** Save the value at slot i so that it can later be used as a + * pivot, see {@link #comparePivot(int)}. */ + protected abstract void setPivot(int i); + + /** Compare the pivot with the slot at j, similarly to + * {@link #compare(int, int) compare(i, j)}. */ + protected abstract int comparePivot(int j); +} diff --git a/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java b/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java index 26f7e37dc9f..79a39bfcaa3 100644 --- a/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java +++ b/lucene/core/src/java/org/apache/lucene/util/IntroSorter.java @@ -16,7 +16,6 @@ */ package org.apache.lucene.util; - /** * {@link Sorter} implementation based on a variant of the quicksort algorithm * called introsort: when @@ -38,8 +37,8 @@ public abstract class IntroSorter extends Sorter { } void quicksort(int from, int to, int maxDepth) { - if (to - from < INSERTION_SORT_THRESHOLD) { - insertionSort(from, to); + if (to - from < BINARY_SORT_THRESHOLD) { + binarySort(from, to); return; } else if (--maxDepth < 0) { heapSort(from, to); @@ -84,11 +83,18 @@ public abstract class IntroSorter extends Sorter { quicksort(left + 1, to, maxDepth); } - /** Save the value at slot i so that it can later be used as a - * pivot, see {@link #comparePivot(int)}. */ + // Don't rely on the slow default impl of setPivot/comparePivot since + // quicksort relies on these methods to be fast for good performance + + @Override protected abstract void setPivot(int i); - /** Compare the pivot with the slot at j, similarly to - * {@link #compare(int, int) compare(i, j)}. */ + @Override protected abstract int comparePivot(int j); + + @Override + protected int compare(int i, int j) { + setPivot(i); + return comparePivot(j); + } } diff --git a/lucene/core/src/java/org/apache/lucene/util/MSBRadixSorter.java b/lucene/core/src/java/org/apache/lucene/util/MSBRadixSorter.java index 33f20b63922..f0e2a617a0f 100644 --- a/lucene/core/src/java/org/apache/lucene/util/MSBRadixSorter.java +++ b/lucene/core/src/java/org/apache/lucene/util/MSBRadixSorter.java @@ -38,6 +38,7 @@ public abstract class MSBRadixSorter extends Sorter { // we store one histogram per recursion level private final int[][] histograms = new int[LEVEL_THRESHOLD][]; private final int[] endOffsets = new int[HISTOGRAM_SIZE]; + private final int[] commonPrefix; private final int maxLength; @@ -47,6 +48,7 @@ public abstract class MSBRadixSorter extends Sorter { */ protected MSBRadixSorter(int maxLength) { this.maxLength = maxLength; + this.commonPrefix = new int[Math.min(24, maxLength)]; } /** Return the k-th byte of the entry at index {@code i}, or {@code -1} if @@ -116,14 +118,14 @@ public abstract class MSBRadixSorter extends Sorter { @Override public void sort(int from, int to) { checkRange(from, to); - sort(from, to, 0); + sort(from, to, 0, 0); } - private void sort(int from, int to, int k) { - if (to - from <= LENGTH_THRESHOLD || k >= LEVEL_THRESHOLD) { + private void sort(int from, int to, int k, int l) { + if (to - from <= LENGTH_THRESHOLD || l >= LEVEL_THRESHOLD) { introSort(from, to, k); } else { - radixSort(from, to, k); + radixSort(from, to, k, l); } } @@ -131,28 +133,30 @@ public abstract class MSBRadixSorter extends Sorter { getFallbackSorter(k).sort(from, to); } - private void radixSort(int from, int to, int k) { - int[] histogram = histograms[k]; + /** + * @param k the character number to compare + * @param l the level of recursion + */ + private void radixSort(int from, int to, int k, int l) { + int[] histogram = histograms[l]; if (histogram == null) { - histogram = histograms[k] = new int[HISTOGRAM_SIZE]; + histogram = histograms[l] = new int[HISTOGRAM_SIZE]; } else { Arrays.fill(histogram, 0); } - buildHistogram(from, to, k, histogram); - - // short-circuit: if all keys have the same byte at offset k, then recurse directly - for (int i = 0; i < HISTOGRAM_SIZE; ++i) { - if (histogram[i] == to - from) { - // everything is in the same bucket, recurse - if (i > 0) { - sort(from, to, k + 1); - } - return; - } else if (histogram[i] != 0) { - break; + final int commonPrefixLength = computeCommonPrefixLengthAndBuildHistogram(from, to, k, histogram); + if (commonPrefixLength > 0) { + // if there are no more chars to compare or if all entries fell into the + // first bucket (which means strings are shorter than k) then we are done + // otherwise recurse + if (k + commonPrefixLength < maxLength + && histogram[0] < to - from) { + radixSort(from, to, k + commonPrefixLength, l); } + return; } + assert assertHistogram(commonPrefixLength, histogram); int[] startOffsets = histogram; int[] endOffsets = this.endOffsets; @@ -167,24 +171,83 @@ public abstract class MSBRadixSorter extends Sorter { int h = endOffsets[i]; final int bucketLen = h - prev; if (bucketLen > 1) { - sort(from + prev, from + h, k + 1); + sort(from + prev, from + h, k + 1, l + 1); } prev = h; } } } + // only used from assert + private boolean assertHistogram(int commonPrefixLength, int[] histogram) { + int numberOfUniqueBytes = 0; + for (int freq : histogram) { + if (freq > 0) { + numberOfUniqueBytes++; + } + } + if (numberOfUniqueBytes == 1) { + assert commonPrefixLength >= 1; + } else { + assert commonPrefixLength == 0 : commonPrefixLength; + } + return true; + } + /** Return a number for the k-th character between 0 and {@link #HISTOGRAM_SIZE}. */ private int getBucket(int i, int k) { return byteAt(i, k) + 1; } - /** Build a histogram of the number of values per {@link #getBucket(int, int) bucket}. */ - private int[] buildHistogram(int from, int to, int k, int[] histogram) { + /** Build a histogram of the number of values per {@link #getBucket(int, int) bucket} + * and return a common prefix length for all visited values. + * @see #buildHistogram */ + private int computeCommonPrefixLengthAndBuildHistogram(int from, int to, int k, int[] histogram) { + final int[] commonPrefix = this.commonPrefix; + int commonPrefixLength = Math.min(commonPrefix.length, maxLength - k); + for (int j = 0; j < commonPrefixLength; ++j) { + final int b = byteAt(from, k + j); + commonPrefix[j] = b; + if (b == -1) { + commonPrefixLength = j + 1; + break; + } + } + + int i; + outer: for (i = from + 1; i < to; ++i) { + for (int j = 0; j < commonPrefixLength; ++j) { + final int b = byteAt(i, k + j); + if (b != commonPrefix[j]) { + commonPrefixLength = j; + if (commonPrefixLength == 0) { // we have no common prefix + histogram[commonPrefix[0] + 1] = i - from; + histogram[b + 1] = 1; + break outer; + } + break; + } + } + } + + if (i < to) { + // the loop got broken because there is no common prefix + assert commonPrefixLength == 0; + buildHistogram(i + 1, to, k, histogram); + } else { + assert commonPrefixLength > 0; + histogram[commonPrefix[0] + 1] = to - from; + } + + return commonPrefixLength; + } + + /** Build an histogram of the k-th characters of values occurring between + * offsets {@code from} and {@code to}, using {@link #getBucket}. */ + private void buildHistogram(int from, int to, int k, int[] histogram) { for (int i = from; i < to; ++i) { histogram[getBucket(i, k)]++; } - return histogram; } /** Accumulate values of the histogram so that it does not store counts but diff --git a/lucene/core/src/java/org/apache/lucene/util/RadixSelector.java b/lucene/core/src/java/org/apache/lucene/util/RadixSelector.java new file mode 100644 index 00000000000..c9b6133f96d --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/RadixSelector.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.util; + +import java.util.Arrays; + +/** Radix selector. + *

This implementation works similarly to a MSB radix sort except that it + * only recurses into the sub partition that contains the desired value. + * @lucene.internal */ +public abstract class RadixSelector extends Selector { + + // after that many levels of recursion we fall back to introselect anyway + // this is used as a protection against the fact that radix sort performs + // worse when there are long common prefixes (probably because of cache + // locality) + private static final int LEVEL_THRESHOLD = 8; + // size of histograms: 256 + 1 to indicate that the string is finished + private static final int HISTOGRAM_SIZE = 257; + // buckets below this size will be sorted with introselect + private static final int LENGTH_THRESHOLD = 100; + + // we store one histogram per recursion level + private final int[] histogram = new int[HISTOGRAM_SIZE]; + private final int[] commonPrefix; + + private final int maxLength; + + /** + * Sole constructor. + * @param maxLength the maximum length of keys, pass {@link Integer#MAX_VALUE} if unknown. + */ + protected RadixSelector(int maxLength) { + this.maxLength = maxLength; + this.commonPrefix = new int[Math.min(24, maxLength)]; + } + + /** Return the k-th byte of the entry at index {@code i}, or {@code -1} if + * its length is less than or equal to {@code k}. This may only be called + * with a value of {@code i} between {@code 0} included and + * {@code maxLength} excluded. */ + protected abstract int byteAt(int i, int k); + + /** Get a fall-back selector which may assume that the first {@code d} bytes + * of all compared strings are equal. This fallback selector is used when + * the range becomes narrow or when the maximum level of recursion has + * been exceeded. */ + protected Selector getFallbackSelector(int d) { + return new IntroSelector() { + @Override + protected void swap(int i, int j) { + RadixSelector.this.swap(i, j); + } + + @Override + protected int compare(int i, int j) { + for (int o = d; o < maxLength; ++o) { + final int b1 = byteAt(i, o); + final int b2 = byteAt(j, o); + if (b1 != b2) { + return b1 - b2; + } else if (b1 == -1) { + break; + } + } + return 0; + } + + @Override + protected void setPivot(int i) { + pivot.setLength(0); + for (int o = d; o < maxLength; ++o) { + final int b = byteAt(i, o); + if (b == -1) { + break; + } + pivot.append((byte) b); + } + } + + @Override + protected int comparePivot(int j) { + for (int o = 0; o < pivot.length(); ++o) { + final int b1 = pivot.byteAt(o) & 0xff; + final int b2 = byteAt(j, d + o); + if (b1 != b2) { + return b1 - b2; + } + } + if (d + pivot.length() == maxLength) { + return 0; + } + return -1 - byteAt(j, d + pivot.length()); + } + + private final BytesRefBuilder pivot = new BytesRefBuilder(); + }; + } + + @Override + public void select(int from, int to, int k) { + checkArgs(from, to, k); + select(from, to, k, 0, 0); + } + + private void select(int from, int to, int k, int d, int l) { + if (to - from <= LENGTH_THRESHOLD || d >= LEVEL_THRESHOLD) { + getFallbackSelector(d).select(from, to, k); + } else { + radixSelect(from, to, k, d, l); + } + } + + /** + * @param d the character number to compare + * @param l the level of recursion + */ + private void radixSelect(int from, int to, int k, int d, int l) { + final int[] histogram = this.histogram; + Arrays.fill(histogram, 0); + + final int commonPrefixLength = computeCommonPrefixLengthAndBuildHistogram(from, to, d, histogram); + if (commonPrefixLength > 0) { + // if there are no more chars to compare or if all entries fell into the + // first bucket (which means strings are shorter than d) then we are done + // otherwise recurse + if (d + commonPrefixLength < maxLength + && histogram[0] < to - from) { + radixSelect(from, to, k, d + commonPrefixLength, l); + } + return; + } + assert assertHistogram(commonPrefixLength, histogram); + + int bucketFrom = from; + for (int bucket = 0; bucket < HISTOGRAM_SIZE; ++bucket) { + final int bucketTo = bucketFrom + histogram[bucket]; + + if (bucketTo > k) { + partition(from, to, bucket, bucketFrom, bucketTo, d); + + if (bucket != 0 && d + 1 < maxLength) { + // all elements in bucket 0 are equal so we only need to recurse if bucket != 0 + select(bucketFrom, bucketTo, k, d + 1, l + 1); + } + return; + } + bucketFrom = bucketTo; + } + throw new AssertionError("Unreachable code"); + } + + // only used from assert + private boolean assertHistogram(int commonPrefixLength, int[] histogram) { + int numberOfUniqueBytes = 0; + for (int freq : histogram) { + if (freq > 0) { + numberOfUniqueBytes++; + } + } + if (numberOfUniqueBytes == 1) { + assert commonPrefixLength >= 1; + } else { + assert commonPrefixLength == 0; + } + return true; + } + + /** Return a number for the k-th character between 0 and {@link #HISTOGRAM_SIZE}. */ + private int getBucket(int i, int k) { + return byteAt(i, k) + 1; + } + + /** Build a histogram of the number of values per {@link #getBucket(int, int) bucket} + * and return a common prefix length for all visited values. + * @see #buildHistogram */ + private int computeCommonPrefixLengthAndBuildHistogram(int from, int to, int k, int[] histogram) { + final int[] commonPrefix = this.commonPrefix; + int commonPrefixLength = Math.min(commonPrefix.length, maxLength - k); + for (int j = 0; j < commonPrefixLength; ++j) { + final int b = byteAt(from, k + j); + commonPrefix[j] = b; + if (b == -1) { + commonPrefixLength = j + 1; + break; + } + } + + int i; + outer: for (i = from + 1; i < to; ++i) { + for (int j = 0; j < commonPrefixLength; ++j) { + final int b = byteAt(i, k + j); + if (b != commonPrefix[j]) { + commonPrefixLength = j; + if (commonPrefixLength == 0) { // we have no common prefix + histogram[commonPrefix[0] + 1] = i - from; + histogram[b + 1] = 1; + break outer; + } + break; + } + } + } + + if (i < to) { + // the loop got broken because there is no common prefix + assert commonPrefixLength == 0; + buildHistogram(i + 1, to, k, histogram); + } else { + assert commonPrefixLength > 0; + histogram[commonPrefix[0] + 1] = to - from; + } + + return commonPrefixLength; + } + + /** Build an histogram of the k-th characters of values occurring between + * offsets {@code from} and {@code to}, using {@link #getBucket}. */ + private void buildHistogram(int from, int to, int k, int[] histogram) { + for (int i = from; i < to; ++i) { + histogram[getBucket(i, k)]++; + } + } + + /** Reorder elements so that all of them that fall into {@code bucket} are + * between offsets {@code bucketFrom} and {@code bucketTo}. */ + private void partition(int from, int to, int bucket, int bucketFrom, int bucketTo, int d) { + int left = from; + int right = to - 1; + + int slot = bucketFrom; + + for (;;) { + int leftBucket = getBucket(left, d); + int rightBucket = getBucket(right, d); + + while (leftBucket <= bucket && left < bucketFrom) { + if (leftBucket == bucket) { + swap(left, slot++); + } else { + ++left; + } + leftBucket = getBucket(left, d); + } + + while (rightBucket >= bucket && right >= bucketTo) { + if (rightBucket == bucket) { + swap(right, slot++); + } else { + --right; + } + rightBucket = getBucket(right, d); + } + + if (left < bucketFrom && right >= bucketTo) { + swap(left++, right--); + } else { + assert left == bucketFrom; + assert right == bucketTo - 1; + break; + } + } + } +} diff --git a/lucene/core/src/java/org/apache/lucene/util/Selector.java b/lucene/core/src/java/org/apache/lucene/util/Selector.java new file mode 100644 index 00000000000..8c17ce4a2f6 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/Selector.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.util; + +/** An implementation of a selection algorithm, ie. computing the k-th greatest + * value from a collection. */ +public abstract class Selector { + + /** Reorder elements so that the element at position {@code k} is the same + * as if all elements were sorted and all other elements are partitioned + * around it: {@code [from, k)} only contains elements that are less than + * or equal to {@code k} and {@code (k, to)} only contains elements that + * are greater than or equal to {@code k}. */ + public abstract void select(int from, int to, int k); + + void checkArgs(int from, int to, int k) { + if (k < from) { + throw new IllegalArgumentException("k must be >= from"); + } + if (k >= to) { + throw new IllegalArgumentException("k must be < to"); + } + } + + /** Swap values at slots i and j. */ + protected abstract void swap(int i, int j); +} diff --git a/lucene/core/src/java/org/apache/lucene/util/Sorter.java b/lucene/core/src/java/org/apache/lucene/util/Sorter.java index 0ac954b0d3b..32d27cd66d5 100644 --- a/lucene/core/src/java/org/apache/lucene/util/Sorter.java +++ b/lucene/core/src/java/org/apache/lucene/util/Sorter.java @@ -23,7 +23,7 @@ import java.util.Comparator; * @lucene.internal */ public abstract class Sorter { - static final int INSERTION_SORT_THRESHOLD = 20; + static final int BINARY_SORT_THRESHOLD = 20; /** Sole constructor, used for inheritance. */ protected Sorter() {} @@ -36,6 +36,20 @@ public abstract class Sorter { /** Swap values at slots i and j. */ protected abstract void swap(int i, int j); + private int pivotIndex; + + /** Save the value at slot i so that it can later be used as a + * pivot, see {@link #comparePivot(int)}. */ + protected void setPivot(int i) { + pivotIndex = i; + } + + /** Compare the pivot with the slot at j, similarly to + * {@link #compare(int, int) compare(i, j)}. */ + protected int comparePivot(int j) { + return compare(pivotIndex, j); + } + /** Sort the slice which starts at from (inclusive) and ends at * to (exclusive). */ public abstract void sort(int from, int to); @@ -163,54 +177,41 @@ public abstract class Sorter { } } - void insertionSort(int from, int to) { - for (int i = from + 1; i < to; ++i) { - for (int j = i; j > from; --j) { - if (compare(j - 1, j) > 0) { - swap(j - 1, j); - } else { - break; - } - } - } - } - + /** + * A binary sort implementation. This performs {@code O(n*log(n))} comparisons + * and {@code O(n^2)} swaps. It is typically used by more sophisticated + * implementations as a fall-back when the numbers of items to sort has become + * less than {@value #BINARY_SORT_THRESHOLD}. + */ void binarySort(int from, int to) { binarySort(from, to, from + 1); } void binarySort(int from, int to, int i) { for ( ; i < to; ++i) { + setPivot(i); int l = from; int h = i - 1; while (l <= h) { final int mid = (l + h) >>> 1; - final int cmp = compare(i, mid); + final int cmp = comparePivot(mid); if (cmp < 0) { h = mid - 1; } else { l = mid + 1; } } - switch (i - l) { - case 2: - swap(l + 1, l + 2); - swap(l, l + 1); - break; - case 1: - swap(l, l + 1); - break; - case 0: - break; - default: - for (int j = i; j > l; --j) { - swap(j - 1, j); - } - break; + for (int j = i; j > l; --j) { + swap(j - 1, j); } } } + /** + * Use heap sort to sort items between {@code from} inclusive and {@code to} + * exclusive. This runs in {@code O(n*log(n))} and is used as a fall-back by + * {@link IntroSorter}. + */ void heapSort(int from, int to) { if (to - from <= 1) { return; diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java b/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java index 42f28edacb0..0dd449c9961 100644 --- a/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java +++ b/lucene/core/src/java/org/apache/lucene/util/automaton/Automaton.java @@ -357,13 +357,13 @@ public class Automaton implements Accountable { } private void growStates() { - if (nextState+2 >= states.length) { + if (nextState+2 > states.length) { states = ArrayUtil.grow(states, nextState+2); } } private void growTransitions() { - if (nextTransition+3 >= transitions.length) { + if (nextTransition+3 > transitions.length) { transitions = ArrayUtil.grow(transitions, nextTransition+3); } } diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java index 97651e47e38..88a84e9efe9 100644 --- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.function.IntFunction; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.MutablePointsReader; import org.apache.lucene.index.MergeState; import org.apache.lucene.index.PointValues.IntersectVisitor; import org.apache.lucene.index.PointValues.Relation; @@ -67,7 +68,7 @@ import org.apache.lucene.util.StringHelper; *

* See this paper for details. * - *

This consumes heap during writing: it allocates a LongBitSet(numPoints), + *

This consumes heap during writing: it allocates a LongBitSet(numPoints), * and then uses up to the specified {@code maxMBSortInHeap} heap space for writing. * *

@@ -111,7 +112,8 @@ public class BKDWriter implements Closeable { final byte[] scratchDiff; final byte[] scratch1; final byte[] scratch2; - final BytesRef scratchBytesRef = new BytesRef(); + final BytesRef scratchBytesRef1 = new BytesRef(); + final BytesRef scratchBytesRef2 = new BytesRef(); final int[] commonPrefixLengths; protected final FixedBitSet docsSeen; @@ -140,10 +142,10 @@ public class BKDWriter implements Closeable { /** True if every document has at most one value. We specialize this case by not bothering to store the ord since it's redundant with docID. */ protected final boolean singleValuePerDoc; - /** How much heap OfflineSorter is allowed to use */ + /** How much heap OfflineSorter is allowed to use */ protected final OfflineSorter.BufferSize offlineSorterBufferMB; - /** How much heap OfflineSorter is allowed to use */ + /** How much heap OfflineSorter is allowed to use */ protected final int offlineSorterMaxTempFiles; private final int maxDoc; @@ -173,7 +175,6 @@ public class BKDWriter implements Closeable { packedBytesLength = numDims * bytesPerDim; scratchDiff = new byte[bytesPerDim]; - scratchBytesRef.length = packedBytesLength; scratch1 = new byte[packedBytesLength]; scratch2 = new byte[packedBytesLength]; commonPrefixLengths = new int[numDims]; @@ -204,7 +205,7 @@ public class BKDWriter implements Closeable { // all recursive halves (i.e. 16 + 8 + 4 + 2) so the memory usage is 2X // what that level would consume, so we multiply by 0.5 to convert from // bytes to points here. Each dimension has its own sorted partition, so - // we must divide by numDims as well. + // we must divide by numDims as wel. maxPointsSortInHeap = (int) (0.5 * (maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc * numDims)); @@ -381,7 +382,7 @@ public class BKDWriter implements Closeable { } else { mappedDocID = docMap.get(oldDocID); } - + if (mappedDocID != -1) { // Not deleted! docID = mappedDocID; @@ -416,15 +417,25 @@ public class BKDWriter implements Closeable { } } - /** More efficient bulk-add for incoming {@link BKDReader}s. This does a merge sort of the already - * sorted values and currently only works when numDims==1. This returns -1 if all documents containing - * dimensional values were deleted. */ - public long merge(IndexOutput out, List docMaps, List readers) throws IOException { - if (numDims != 1) { - throw new UnsupportedOperationException("numDims must be 1 but got " + numDims); + /** Write a field from a {@link MutablePointsReader}. This way of writing + * points is faster than regular writes with {@link BKDWriter#add} since + * there is opportunity for reordering points before writing them to + * disk. This method does not use transient disk in order to reorder points. + */ + public long writeField(IndexOutput out, String fieldName, MutablePointsReader reader) throws IOException { + if (numDims == 1) { + return writeField1Dim(out, fieldName, reader); + } else { + return writeFieldNDims(out, fieldName, reader); } + } + + + /* In the 2+D case, we recursively pick the split dimension, compute the + * median value and partition other values around it. */ + private long writeFieldNDims(IndexOutput out, String fieldName, MutablePointsReader reader) throws IOException { if (pointCount != 0) { - throw new IllegalStateException("cannot mix add and merge"); + throw new IllegalStateException("cannot mix add and writeField"); } // Catch user silliness: @@ -435,6 +446,81 @@ public class BKDWriter implements Closeable { // Mark that we already finished: heapPointWriter = null; + long countPerLeaf = pointCount = reader.size(fieldName); + long innerNodeCount = 1; + + while (countPerLeaf > maxPointsInLeafNode) { + countPerLeaf = (countPerLeaf+1)/2; + innerNodeCount *= 2; + } + + int numLeaves = Math.toIntExact(innerNodeCount); + + checkMaxLeafNodeCount(numLeaves); + + final byte[] splitPackedValues = new byte[numLeaves * (bytesPerDim + 1)]; + final long[] leafBlockFPs = new long[numLeaves]; + + // compute the min/max for this slice + Arrays.fill(minPackedValue, (byte) 0xff); + Arrays.fill(maxPackedValue, (byte) 0); + for (int i = 0; i < Math.toIntExact(pointCount); ++i) { + reader.getValue(i, scratchBytesRef1); + for(int dim=0;dim 0) { + System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset + offset, maxPackedValue, offset, bytesPerDim); + } + } + + docsSeen.set(reader.getDocID(i)); + } + + build(1, numLeaves, reader, 0, Math.toIntExact(pointCount), out, + minPackedValue, maxPackedValue, splitPackedValues, leafBlockFPs, + new int[maxPointsInLeafNode]); + + long indexFP = out.getFilePointer(); + writeIndex(out, leafBlockFPs, splitPackedValues); + return indexFP; + } + + + /* In the 1D case, we can simply sort points in ascending order and use the + * same writing logic as we use at merge time. */ + private long writeField1Dim(IndexOutput out, String fieldName, MutablePointsReader reader) throws IOException { + MutablePointsReaderUtils.sort(maxDoc, packedBytesLength, reader, 0, Math.toIntExact(reader.size(fieldName))); + + final OneDimensionBKDWriter oneDimWriter = new OneDimensionBKDWriter(out); + + reader.intersect(fieldName, new IntersectVisitor() { + + @Override + public void visit(int docID, byte[] packedValue) throws IOException { + oneDimWriter.add(packedValue, docID); + } + + @Override + public void visit(int docID) throws IOException { + throw new IllegalStateException(); + } + + @Override + public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + return Relation.CELL_CROSSES_QUERY; + } + }); + + return oneDimWriter.finish(); + } + + /** More efficient bulk-add for incoming {@link BKDReader}s. This does a merge sort of the already + * sorted values and currently only works when numDims==1. This returns -1 if all documents containing + * dimensional values were deleted. */ + public long merge(IndexOutput out, List docMaps, List readers) throws IOException { assert docMaps == null || readers.size() == docMaps.size(); BKDMergeQueue queue = new BKDMergeQueue(bytesPerDim, readers.size()); @@ -453,72 +539,14 @@ public class BKDWriter implements Closeable { } } - if (queue.size() == 0) { - return -1; - } - - int leafCount = 0; - List leafBlockFPs = new ArrayList<>(); - List leafBlockStartValues = new ArrayList<>(); - - // Target halfway between min and max allowed for the leaf: - int pointsPerLeafBlock = (int) (0.75 * maxPointsInLeafNode); - //System.out.println("POINTS PER: " + pointsPerLeafBlock); - - byte[] lastPackedValue = new byte[bytesPerDim]; - byte[] firstPackedValue = new byte[bytesPerDim]; - long valueCount = 0; - - // Buffer up each leaf block's docs and values - int[] leafBlockDocIDs = new int[maxPointsInLeafNode]; - byte[][] leafBlockPackedValues = new byte[maxPointsInLeafNode][]; - for(int i=0;i 0) { - // Save the first (minimum) value in each leaf block except the first, to build the split value index in the end: - leafBlockStartValues.add(Arrays.copyOf(reader.state.scratchPackedValue, bytesPerDim)); - } - Arrays.fill(commonPrefixLengths, bytesPerDim); - System.arraycopy(reader.state.scratchPackedValue, 0, firstPackedValue, 0, bytesPerDim); - } else { - // Find per-dim common prefix: - for(int dim=0;dim packedValues = new IntFunction() { - final BytesRef scratch = new BytesRef(); + final IndexOutput out; + final List leafBlockFPs = new ArrayList<>(); + final List leafBlockStartValues = new ArrayList<>(); + final byte[] leafValues = new byte[maxPointsInLeafNode * packedBytesLength]; + final int[] leafDocs = new int[maxPointsInLeafNode]; + long valueCount; + int leafCount; - { - scratch.length = packedBytesLength; - scratch.offset = 0; - } + OneDimensionBKDWriter(IndexOutput out) { + if (numDims != 1) { + throw new UnsupportedOperationException("numDims must be 1 but got " + numDims); + } + if (pointCount != 0) { + throw new IllegalStateException("cannot mix add and merge"); + } - @Override - public BytesRef apply(int i) { - scratch.bytes = leafBlockPackedValues[i]; - return scratch; - } - }; - writeLeafBlockPackedValues(out, commonPrefixLengths, leafCount, 0, packedValues); + // Catch user silliness: + if (heapPointWriter == null && tempInput == null) { + throw new IllegalStateException("already finished"); + } + // Mark that we already finished: + heapPointWriter = null; + + this.out = out; + + lastPackedValue = new byte[packedBytesLength]; + } + + // for asserts + final byte[] lastPackedValue; + int lastDocID; + + void add(byte[] packedValue, int docID) throws IOException { + assert valueInOrder(valueCount + leafCount, + 0, lastPackedValue, packedValue, 0, docID, lastDocID); + + System.arraycopy(packedValue, 0, leafValues, leafCount * packedBytesLength, packedBytesLength); + leafDocs[leafCount] = docID; + docsSeen.set(docID); + leafCount++; + + if (valueCount > totalPointCount) { + throw new IllegalStateException("totalPointCount=" + totalPointCount + " was passed when we were created, but we just hit " + pointCount + " values"); + } + + if (leafCount == maxPointsInLeafNode) { + // We write a block once we hit exactly the max count ... this is different from + // when we flush a new segment, where we write between max/2 and max per leaf block, + // so merged segments will behave differently from newly flushed segments: + writeLeafBlock(); leafCount = 0; } + + assert (lastDocID = docID) >= 0; // only assign when asserts are enabled } - pointCount = valueCount; + public long finish() throws IOException { + if (leafCount > 0) { + writeLeafBlock(); + leafCount = 0; + } - long indexFP = out.getFilePointer(); + if (valueCount == 0) { + return -1; + } - int numInnerNodes = leafBlockStartValues.size(); + pointCount = valueCount; - //System.out.println("BKDW: now rotate numInnerNodes=" + numInnerNodes + " leafBlockStarts=" + leafBlockStartValues.size()); + long indexFP = out.getFilePointer(); - byte[] index = new byte[(1+numInnerNodes) * (1+bytesPerDim)]; - rotateToTree(1, 0, numInnerNodes, index, leafBlockStartValues); - long[] arr = new long[leafBlockFPs.size()]; - for(int i=0;i>> Math.max(0, shift)) & 0xff; + } + } + + @Override + protected org.apache.lucene.util.Sorter getFallbackSorter(int k) { + return new IntroSorter() { + + final BytesRef pivot = new BytesRef(); + final BytesRef scratch = new BytesRef(); + int pivotDoc; + + @Override + protected void swap(int i, int j) { + reader.swap(i, j); + } + + @Override + protected void setPivot(int i) { + reader.getValue(i, pivot); + pivotDoc = reader.getDocID(i); + } + + @Override + protected int comparePivot(int j) { + if (k < packedBytesLength) { + reader.getValue(j, scratch); + int cmp = StringHelper.compare(packedBytesLength - k, pivot.bytes, pivot.offset + k, scratch.bytes, scratch.offset + k); + if (cmp != 0) { + return cmp; + } + } + return pivotDoc - reader.getDocID(j); + } + }; + } + + }.sort(from, to); + } + + /** Sort points on the given dimension. */ + static void sortByDim(int sortedDim, int bytesPerDim, int[] commonPrefixLengths, + MutablePointsReader reader, int from, int to, + BytesRef scratch1, BytesRef scratch2) { + + // No need for a fancy radix sort here, this is called on the leaves only so + // there are not many values to sort + final int offset = sortedDim * bytesPerDim + commonPrefixLengths[sortedDim]; + final int numBytesToCompare = bytesPerDim - commonPrefixLengths[sortedDim]; + new IntroSorter() { + + final BytesRef pivot = scratch1; + int pivotDoc = -1; + + @Override + protected void swap(int i, int j) { + reader.swap(i, j); + } + + @Override + protected void setPivot(int i) { + reader.getValue(i, pivot); + pivotDoc = reader.getDocID(i); + } + + @Override + protected int comparePivot(int j) { + reader.getValue(j, scratch2); + int cmp = StringHelper.compare(numBytesToCompare, pivot.bytes, pivot.offset + offset, scratch2.bytes, scratch2.offset + offset); + if (cmp == 0) { + cmp = pivotDoc - reader.getDocID(j); + } + return cmp; + } + }.sort(from, to); + } + + /** Partition points around {@code mid}. All values on the left must be less + * than or equal to it and all values on the right must be greater than or + * equal to it. */ + static void partition(int maxDoc, int splitDim, int bytesPerDim, int commonPrefixLen, + MutablePointsReader reader, int from, int to, int mid, + BytesRef scratch1, BytesRef scratch2) { + final int offset = splitDim * bytesPerDim + commonPrefixLen; + final int cmpBytes = bytesPerDim - commonPrefixLen; + final int bitsPerDocId = PackedInts.bitsRequired(maxDoc - 1); + new RadixSelector(cmpBytes + (bitsPerDocId + 7) / 8) { + + @Override + protected Selector getFallbackSelector(int k) { + return new IntroSelector() { + + final BytesRef pivot = scratch1; + int pivotDoc; + + @Override + protected void swap(int i, int j) { + reader.swap(i, j); + } + + @Override + protected void setPivot(int i) { + reader.getValue(i, pivot); + pivotDoc = reader.getDocID(i); + } + + @Override + protected int comparePivot(int j) { + if (k < cmpBytes) { + reader.getValue(j, scratch2); + int cmp = StringHelper.compare(cmpBytes - k, pivot.bytes, pivot.offset + offset + k, scratch2.bytes, scratch2.offset + offset + k); + if (cmp != 0) { + return cmp; + } + } + return pivotDoc - reader.getDocID(j); + } + }; + } + + @Override + protected void swap(int i, int j) { + reader.swap(i, j); + } + + @Override + protected int byteAt(int i, int k) { + if (k < cmpBytes) { + return Byte.toUnsignedInt(reader.getByteAt(i, offset + k)); + } else { + final int shift = bitsPerDocId - ((k - cmpBytes + 1) << 3); + return (reader.getDocID(i) >>> Math.max(0, shift)) & 0xff; + } + } + }.select(from, to, mid); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointsFormat.java index 4b898c30e21..afa8ec412f5 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointsFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointsFormat.java @@ -41,8 +41,9 @@ public class TestLucene60PointsFormat extends BasePointsFormatTestCase { if (random().nextBoolean()) { // randomize parameters int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 500); + double maxMBSortInHeap = 3.0 + (3*random().nextDouble()); if (VERBOSE) { - System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode); + System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap); } // sneaky impersonation! @@ -52,7 +53,7 @@ public class TestLucene60PointsFormat extends BasePointsFormatTestCase { return new PointsFormat() { @Override public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException { - return new Lucene60PointsWriter(writeState, maxPointsInLeafNode); + return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java index c139b64d9a5..cf8372dc427 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java @@ -1156,8 +1156,9 @@ public class TestPointQueries extends LuceneTestCase { private static Codec getCodec() { if (Codec.getDefault().getName().equals("Lucene62")) { int maxPointsInLeafNode = TestUtil.nextInt(random(), 16, 2048); + double maxMBSortInHeap = 5.0 + (3*random().nextDouble()); if (VERBOSE) { - System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode); + System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap); } return new FilterCodec("Lucene62", Codec.getDefault()) { @@ -1166,7 +1167,7 @@ public class TestPointQueries extends LuceneTestCase { return new PointsFormat() { @Override public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException { - return new Lucene60PointsWriter(writeState, maxPointsInLeafNode); + return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/util/TestByteBlockPool.java b/lucene/core/src/test/org/apache/lucene/util/TestByteBlockPool.java index b425b761877..df73687d17c 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestByteBlockPool.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestByteBlockPool.java @@ -45,7 +45,26 @@ public class TestByteBlockPool extends LuceneTestCase { for (BytesRef expected : list) { ref.grow(expected.length); ref.setLength(expected.length); - pool.readBytes(position, ref.bytes(), 0, ref.length()); + switch (random().nextInt(3)) { + case 0: + // copy bytes + pool.readBytes(position, ref.bytes(), 0, ref.length()); + break; + case 1: + // copy bytes one by one + for (int i = 0; i < ref.length(); ++i) { + ref.setByteAt(i, pool.readByte(position + i)); + } + break; + case 2: + BytesRef scratch = new BytesRef(); + scratch.length = ref.length(); + pool.setRawBytesRef(scratch, position); + System.arraycopy(scratch.bytes, scratch.offset, ref.bytes(), 0, ref.length()); + break; + default: + fail(); + } assertEquals(expected, ref.get()); position += ref.length(); } diff --git a/lucene/core/src/test/org/apache/lucene/util/TestIntroSelector.java b/lucene/core/src/test/org/apache/lucene/util/TestIntroSelector.java new file mode 100644 index 00000000000..468b2f78809 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/util/TestIntroSelector.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.util; + +import java.util.Arrays; + +public class TestIntroSelector extends LuceneTestCase { + + public void testSelect() { + for (int iter = 0; iter < 100; ++iter) { + doTestSelect(false); + } + } + + public void testSlowSelect() { + for (int iter = 0; iter < 100; ++iter) { + doTestSelect(true); + } + } + + private void doTestSelect(boolean slow) { + final int from = random().nextInt(5); + final int to = from + TestUtil.nextInt(random(), 1, 10000); + final int max = random().nextBoolean() ? random().nextInt(100) : random().nextInt(100000); + Integer[] arr = new Integer[from + to + random().nextInt(5)]; + for (int i = 0; i < arr.length; ++i) { + arr[i] = TestUtil.nextInt(random(), 0, max); + } + final int k = TestUtil.nextInt(random(), from, to - 1); + + Integer[] expected = arr.clone(); + Arrays.sort(expected, from, to); + + Integer[] actual = arr.clone(); + IntroSelector selector = new IntroSelector() { + + Integer pivot; + + @Override + protected void swap(int i, int j) { + ArrayUtil.swap(actual, i, j); + } + + @Override + protected void setPivot(int i) { + pivot = actual[i]; + } + + @Override + protected int comparePivot(int j) { + return pivot.compareTo(actual[j]); + } + }; + if (slow) { + selector.slowSelect(from, to, k); + } else { + selector.select(from, to, k); + } + + assertEquals(expected[k], actual[k]); + for (int i = 0; i < actual.length; ++i) { + if (i < from || i >= to) { + assertSame(arr[i], actual[i]); + } else if (i <= k) { + assertTrue(actual[i].intValue() <= actual[k].intValue()); + } else { + assertTrue(actual[i].intValue() >= actual[k].intValue()); + } + } + } + +} diff --git a/lucene/core/src/test/org/apache/lucene/util/TestMSBRadixSorter.java b/lucene/core/src/test/org/apache/lucene/util/TestMSBRadixSorter.java index bc5af7f23fe..52eb494711a 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestMSBRadixSorter.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestMSBRadixSorter.java @@ -17,6 +17,8 @@ package org.apache.lucene.util; import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; public class TestMSBRadixSorter extends LuceneTestCase { @@ -41,9 +43,12 @@ public class TestMSBRadixSorter extends LuceneTestCase { break; } + final int finalMaxLength = maxLength; new MSBRadixSorter(maxLength) { + @Override protected int byteAt(int i, int k) { + assertTrue(k < finalMaxLength); BytesRef ref = refs[i]; if (ref.length <= k) { return -1; @@ -114,4 +119,67 @@ public class TestMSBRadixSorter extends LuceneTestCase { testRandom(TestUtil.nextInt(random(), 1, 30), 2); } } + + public void testRandom2() { + // how large our alphabet is + int letterCount = TestUtil.nextInt(random(), 2, 10); + + // how many substring fragments to use + int substringCount = TestUtil.nextInt(random(), 2, 10); + Set substringsSet = new HashSet<>(); + + // how many strings to make + int stringCount = atLeast(10000); + + //System.out.println("letterCount=" + letterCount + " substringCount=" + substringCount + " stringCount=" + stringCount); + while(substringsSet.size() < substringCount) { + int length = TestUtil.nextInt(random(), 2, 10); + byte[] bytes = new byte[length]; + for(int i=0;i stringsSet = new HashSet<>(); + int iters = 0; + while (stringsSet.size() < stringCount && iters < stringCount*5) { + int count = TestUtil.nextInt(random(), 1, 5); + BytesRefBuilder b = new BytesRefBuilder(); + for(int i=0;i= v) { + b.append(substrings[j]); + break; + } + } + } + BytesRef br = b.toBytesRef(); + stringsSet.add(br); + //System.out.println("add string count=" + stringsSet.size() + ": " + br); + iters++; + } + + test(stringsSet.toArray(new BytesRef[stringsSet.size()]), stringsSet.size()); + } } diff --git a/lucene/core/src/test/org/apache/lucene/util/TestRadixSelector.java b/lucene/core/src/test/org/apache/lucene/util/TestRadixSelector.java new file mode 100644 index 00000000000..30165802643 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/util/TestRadixSelector.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.util; + +import java.util.Arrays; + +public class TestRadixSelector extends LuceneTestCase { + + public void testSelect() { + for (int iter = 0; iter < 100; ++iter) { + doTestSelect(); + } + } + + private void doTestSelect() { + final int from = random().nextInt(5); + final int to = from + TestUtil.nextInt(random(), 1, 10000); + final int maxLen = TestUtil.nextInt(random(), 1, 12); + BytesRef[] arr = new BytesRef[from + to + random().nextInt(5)]; + for (int i = 0; i < arr.length; ++i) { + byte[] bytes = new byte[TestUtil.nextInt(random(), 0, maxLen)]; + random().nextBytes(bytes); + arr[i] = new BytesRef(bytes); + } + doTest(arr, from, to, maxLen); + } + + public void testSharedPrefixes() { + for (int iter = 0; iter < 100; ++iter) { + doTestSharedPrefixes(); + } + } + + private void doTestSharedPrefixes() { + final int from = random().nextInt(5); + final int to = from + TestUtil.nextInt(random(), 1, 10000); + final int maxLen = TestUtil.nextInt(random(), 1, 12); + BytesRef[] arr = new BytesRef[from + to + random().nextInt(5)]; + for (int i = 0; i < arr.length; ++i) { + byte[] bytes = new byte[TestUtil.nextInt(random(), 0, maxLen)]; + random().nextBytes(bytes); + arr[i] = new BytesRef(bytes); + } + final int sharedPrefixLength = Math.min(arr[0].length, TestUtil.nextInt(random(), 1, maxLen)); + for (int i = 1; i < arr.length; ++i) { + System.arraycopy(arr[0].bytes, arr[0].offset, arr[i].bytes, arr[i].offset, Math.min(sharedPrefixLength, arr[i].length)); + } + doTest(arr, from, to, maxLen); + } + + private void doTest(BytesRef[] arr, int from, int to, int maxLen) { + final int k = TestUtil.nextInt(random(), from, to - 1); + + BytesRef[] expected = arr.clone(); + Arrays.sort(expected, from, to); + + BytesRef[] actual = arr.clone(); + final int enforcedMaxLen = random().nextBoolean() ? maxLen : Integer.MAX_VALUE; + RadixSelector selector = new RadixSelector(enforcedMaxLen) { + + @Override + protected void swap(int i, int j) { + ArrayUtil.swap(actual, i, j); + } + + @Override + protected int byteAt(int i, int k) { + assertTrue(k < enforcedMaxLen); + BytesRef b = actual[i]; + if (k >= b.length) { + return -1; + } else { + return Byte.toUnsignedInt(b.bytes[b.offset + k]); + } + } + + }; + selector.select(from, to, k); + + assertEquals(expected[k], actual[k]); + for (int i = 0; i < actual.length; ++i) { + if (i < from || i >= to) { + assertSame(arr[i], actual[i]); + } else if (i <= k) { + assertTrue(actual[i].compareTo(actual[k]) <= 0); + } else { + assertTrue(actual[i].compareTo(actual[k]) >= 0); + } + } + } + +} diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestMutablePointsReaderUtils.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestMutablePointsReaderUtils.java new file mode 100644 index 00000000000..4616ce3901d --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestMutablePointsReaderUtils.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.util.bkd; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; + +import org.apache.lucene.codecs.MutablePointsReader; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.StringHelper; +import org.apache.lucene.util.TestUtil; + +public class TestMutablePointsReaderUtils extends LuceneTestCase { + + public void testSort() { + for (int iter = 0; iter < 5; ++iter) { + doTestSort(); + } + } + + private void doTestSort() { + final int bytesPerDim = TestUtil.nextInt(random(), 1, 16); + final int maxDoc = TestUtil.nextInt(random(), 1, 1 << random().nextInt(30)); + Point[] points = createRandomPoints(1, bytesPerDim, maxDoc); + DummyPointsReader reader = new DummyPointsReader(points); + MutablePointsReaderUtils.sort(maxDoc, bytesPerDim, reader, 0, points.length); + Arrays.sort(points, new Comparator() { + @Override + public int compare(Point o1, Point o2) { + int cmp = o1.packedValue.compareTo(o2.packedValue); + if (cmp == 0) { + cmp = Integer.compare(o1.doc, o2.doc); + } + return cmp; + } + }); + assertNotSame(points, reader.points); + assertArrayEquals(points, reader.points); + } + + public void testSortByDim() { + for (int iter = 0; iter < 5; ++iter) { + doTestSortByDim(); + } + } + + private void doTestSortByDim() { + final int numDims = TestUtil.nextInt(random(), 1, 8); + final int bytesPerDim = TestUtil.nextInt(random(), 1, 16); + final int maxDoc = TestUtil.nextInt(random(), 1, 1 << random().nextInt(30)); + Point[] points = createRandomPoints(numDims, bytesPerDim, maxDoc); + int[] commonPrefixLengths = new int[numDims]; + for (int i = 0; i < commonPrefixLengths.length; ++i) { + commonPrefixLengths[i] = TestUtil.nextInt(random(), 0, bytesPerDim); + } + BytesRef firstValue = points[0].packedValue; + for (int i = 1; i < points.length; ++i) { + for (int dim = 0; dim < numDims; ++dim) { + int offset = dim * bytesPerDim; + BytesRef packedValue = points[i].packedValue; + System.arraycopy(firstValue.bytes, firstValue.offset + offset, packedValue.bytes, packedValue.offset + offset, commonPrefixLengths[dim]); + } + } + DummyPointsReader reader = new DummyPointsReader(points); + final int sortedDim = random().nextInt(numDims); + MutablePointsReaderUtils.sortByDim(sortedDim, bytesPerDim, commonPrefixLengths, reader, 0, points.length, + new BytesRef(), new BytesRef()); + for (int i = 1; i < points.length; ++i) { + final int offset = sortedDim * bytesPerDim; + BytesRef previousValue = reader.points[i-1].packedValue; + BytesRef currentValue = reader.points[i].packedValue; + int cmp = StringHelper.compare(bytesPerDim, + previousValue.bytes, previousValue.offset + offset, + currentValue.bytes, currentValue.offset + offset); + if (cmp == 0) { + cmp = reader.points[i - 1].doc - reader.points[i].doc; + } + assertTrue(cmp <= 0); + } + } + + public void testPartition() { + for (int iter = 0; iter < 5; ++iter) { + doTestPartition(); + } + } + + private void doTestPartition() { + final int numDims = TestUtil.nextInt(random(), 1, 8); + final int bytesPerDim = TestUtil.nextInt(random(), 1, 16); + final int maxDoc = TestUtil.nextInt(random(), 1, 1 << random().nextInt(30)); + Point[] points = createRandomPoints(numDims, bytesPerDim, maxDoc); + int commonPrefixLength = TestUtil.nextInt(random(), 0, bytesPerDim); + final int splitDim = random().nextInt(numDims); + BytesRef firstValue = points[0].packedValue; + for (int i = 1; i < points.length; ++i) { + BytesRef packedValue = points[i].packedValue; + int offset = splitDim * bytesPerDim; + System.arraycopy(firstValue.bytes, firstValue.offset + offset, packedValue.bytes, packedValue.offset + offset, commonPrefixLength); + } + DummyPointsReader reader = new DummyPointsReader(points); + final int pivot = TestUtil.nextInt(random(), 0, points.length - 1); + MutablePointsReaderUtils.partition(maxDoc, splitDim, bytesPerDim, commonPrefixLength, reader, 0, points.length, pivot, + new BytesRef(), new BytesRef()); + BytesRef pivotValue = reader.points[pivot].packedValue; + int offset = splitDim * bytesPerDim; + for (int i = 0; i < points.length; ++i) { + BytesRef value = reader.points[i].packedValue; + int cmp = StringHelper.compare(bytesPerDim, + value.bytes, value.offset + offset, + pivotValue.bytes, pivotValue.offset + offset); + if (cmp == 0) { + cmp = reader.points[i].doc - reader.points[pivot].doc; + } + if (i < pivot) { + assertTrue(cmp <= 0); + } else if (i > pivot) { + assertTrue(cmp >= 0); + } else { + assertEquals(0, cmp); + } + } + } + + private static Point[] createRandomPoints(int numDims, int bytesPerDim, int maxDoc) { + final int packedBytesLength = numDims * bytesPerDim; + final int numPoints = TestUtil.nextInt(random(), 1, 100000); + Point[] points = new Point[numPoints]; + for (int i = 0; i < numPoints; ++i) { + byte[] value = new byte[packedBytesLength]; + random().nextBytes(value); + points[i] = new Point(value, random().nextInt(maxDoc)); + } + return points; + } + + private static class Point { + final BytesRef packedValue; + final int doc; + + Point(byte[] packedValue, int doc) { + // use a non-null offset to make sure MutablePointsReaderUtils does not ignore it + this.packedValue = new BytesRef(packedValue.length + 1); + this.packedValue.bytes[0] = (byte) random().nextInt(256); + this.packedValue.offset = 1; + this.packedValue.length = packedValue.length; + this.doc = doc; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj instanceof Point == false) { + return false; + } + Point that = (Point) obj; + return packedValue.equals(that.packedValue) && doc == that.doc; + } + + @Override + public int hashCode() { + return 31 * packedValue.hashCode() + doc; + } + + @Override + public String toString() { + return "value=" + packedValue + " doc=" + doc; + } + } + + private static class DummyPointsReader extends MutablePointsReader { + + private final Point[] points; + + DummyPointsReader(Point[] points) { + this.points = points.clone(); + } + + @Override + public void close() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void getValue(int i, BytesRef packedValue) { + packedValue.bytes = points[i].packedValue.bytes; + packedValue.offset = points[i].packedValue.offset; + packedValue.length = points[i].packedValue.length; + } + + @Override + public byte getByteAt(int i, int k) { + BytesRef packedValue = points[i].packedValue; + return packedValue.bytes[packedValue.offset + k]; + } + + @Override + public int getDocID(int i) { + return points[i].doc; + } + + @Override + public void swap(int i, int j) { + ArrayUtil.swap(points, i, j); + } + + @Override + public void checkIntegrity() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void intersect(String fieldName, IntersectVisitor visitor) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] getMinPackedValue(String fieldName) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] getMaxPackedValue(String fieldName) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int getNumDimensions(String fieldName) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int getBytesPerDimension(String fieldName) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long size(String fieldName) { + throw new UnsupportedOperationException(); + } + + @Override + public int getDocCount(String fieldName) { + throw new UnsupportedOperationException(); + } + + } + +} diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index 510befa4ab7..bc46ee6ccac 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -106,6 +106,7 @@ io.netty.netty-all.version = 4.0.36.Final org.apache.curator.version = 2.8.0 /org.apache.curator/curator-client = ${org.apache.curator.version} /org.apache.curator/curator-framework = ${org.apache.curator.version} +/org.apache.curator/curator-recipes = ${org.apache.curator.version} /org.apache.derby/derby = 10.9.1.0 diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ComparisonBoolFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ComparisonBoolFunction.java new file mode 100644 index 00000000000..82d723a57d5 --- /dev/null +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ComparisonBoolFunction.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.queries.function.valuesource; + +import java.io.IOException; +import java.util.Map; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.docvalues.BoolDocValues; +import org.apache.lucene.search.IndexSearcher; + +/** + * Base class for comparison operators useful within an "if"/conditional. + */ +public abstract class ComparisonBoolFunction extends BoolFunction { + + private final ValueSource lhs; + private final ValueSource rhs; + private final String name; + + public ComparisonBoolFunction(ValueSource lhs, ValueSource rhs, String name) { + this.lhs = lhs; + this.rhs = rhs; + this.name = name; + } + + /** Perform the comparison, returning true or false */ + public abstract boolean compare(int doc, FunctionValues lhs, FunctionValues rhs); + + /** Uniquely identify the operation (ie "gt", "lt" "gte", etc) */ + public String name() { + return this.name; + } + + @Override + public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException { + final FunctionValues lhsVal = this.lhs.getValues(context, readerContext); + final FunctionValues rhsVal = this.rhs.getValues(context, readerContext); + final String compLabel = this.name(); + + return new BoolDocValues(this) { + @Override + public boolean boolVal(int doc) { + return compare(doc, lhsVal, rhsVal); + } + + @Override + public String toString(int doc) { + return compLabel + "(" + lhsVal.toString(doc) + "," + rhsVal.toString(doc) + ")"; + } + + @Override + public boolean exists(int doc) { + return lhsVal.exists(doc) && rhsVal.exists(doc); + } + + }; + } + + @Override + public boolean equals(Object o) { + if (this.getClass() != o.getClass()) return false; + ComparisonBoolFunction other = (ComparisonBoolFunction)o; + return name().equals(other.name()) + && lhs.equals(other.lhs) + && rhs.equals(other.rhs); } + + @Override + public int hashCode() { + int h = this.getClass().hashCode(); + h = h * 31 + this.name().hashCode(); + h = h * 31 + lhs.hashCode(); + h = h * 31 + rhs.hashCode(); + return h; + } + + @Override + public String description() { + return name() + "(" + lhs.description() + "," + rhs.description() + ")"; + } + + @Override + public void createWeight(Map context, IndexSearcher searcher) throws IOException { + lhs.createWeight(context, searcher); + rhs.createWeight(context, searcher); + } + +} diff --git a/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/FileMetaData.java b/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/FileMetaData.java index ac3e1f0be29..5d63a8ca039 100644 --- a/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/FileMetaData.java +++ b/lucene/replicator/src/java/org/apache/lucene/replicator/nrt/FileMetaData.java @@ -38,5 +38,10 @@ public class FileMetaData { this.length = length; this.checksum = checksum; } + + @Override + public String toString() { + return "FileMetaData(length=" + length + ")"; + } } diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleCopyJob.java b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleCopyJob.java index ba72ae4b858..7e148810b47 100644 --- a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleCopyJob.java +++ b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleCopyJob.java @@ -118,6 +118,8 @@ class SimpleCopyJob extends CopyJob { return highPriority ? -1 : 1; } else if (ord < other.ord) { return -1; + } else if (ord > other.ord) { + return 1; } else { return 0; } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Bounds.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Bounds.java index 866068a0b75..97a5d13a232 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Bounds.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Bounds.java @@ -121,4 +121,11 @@ public interface Bounds { */ public Bounds noBottomLatitudeBound(); + /** Signal that there is no bound whatsoever. + * The bound is limited only by the constraints of the + * planet. + *@return the updated Bounds object., + */ + public Bounds noBound(final PlanetModel planetModel); + } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LatLonBounds.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LatLonBounds.java index a6072435854..ef43dccd452 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LatLonBounds.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/LatLonBounds.java @@ -253,6 +253,11 @@ public class LatLonBounds implements Bounds { return this; } + @Override + public Bounds noBound(final PlanetModel planetModel) { + return noLongitudeBound().noTopLatitudeBound().noBottomLatitudeBound(); + } + // Protected methods /** Update latitude bound. diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java index 47c1b30b4d5..5917ce7af3c 100755 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java @@ -1003,13 +1003,14 @@ public class Plane extends Vector { * D - MINIMUM_RESOLUTION. Both are examined and intersection points determined. */ protected void findIntersectionBounds(final PlanetModel planetModel, final Bounds boundsInfo, final Plane q, final Membership... bounds) { + //System.out.println("Finding intersection bounds"); // Unnormalized, unchecked... final double lineVectorX = y * q.z - z * q.y; final double lineVectorY = z * q.x - x * q.z; final double lineVectorZ = x * q.y - y * q.x; if (Math.abs(lineVectorX) < MINIMUM_RESOLUTION && Math.abs(lineVectorY) < MINIMUM_RESOLUTION && Math.abs(lineVectorZ) < MINIMUM_RESOLUTION) { // Degenerate case: parallel planes - //System.err.println(" planes are parallel - no intersection"); + //System.out.println(" planes are parallel - no intersection"); return; } @@ -1037,9 +1038,10 @@ public class Plane extends Vector { final double denomXZ = this.x * q.z - this.z * q.x; final double denomXY = this.x * q.y - this.y * q.x; if (Math.abs(denomYZ) >= Math.abs(denomXZ) && Math.abs(denomYZ) >= Math.abs(denomXY)) { + //System.out.println("X biggest"); // X is the biggest, so our point will have x0 = 0.0 if (Math.abs(denomYZ) < MINIMUM_RESOLUTION_SQUARED) { - //System.err.println(" Denominator is zero: no intersection"); + //System.out.println(" Denominator is zero: no intersection"); return; } final double denom = 1.0 / denomYZ; @@ -1061,9 +1063,10 @@ public class Plane extends Vector { 0.0, (-(this.D-MINIMUM_RESOLUTION) * q.z - this.z * -(q.D-MINIMUM_RESOLUTION)) * denom, (this.y * -(q.D-MINIMUM_RESOLUTION) + (this.D-MINIMUM_RESOLUTION) * q.y) * denom, bounds); } else if (Math.abs(denomXZ) >= Math.abs(denomXY) && Math.abs(denomXZ) >= Math.abs(denomYZ)) { + //System.out.println("Y biggest"); // Y is the biggest, so y0 = 0.0 if (Math.abs(denomXZ) < MINIMUM_RESOLUTION_SQUARED) { - //System.err.println(" Denominator is zero: no intersection"); + //System.out.println(" Denominator is zero: no intersection"); return; } final double denom = 1.0 / denomXZ; @@ -1084,9 +1087,10 @@ public class Plane extends Vector { (-(this.D-MINIMUM_RESOLUTION) * q.z - this.z * -(q.D-MINIMUM_RESOLUTION)) * denom, 0.0, (this.x * -(q.D-MINIMUM_RESOLUTION) + (this.D-MINIMUM_RESOLUTION) * q.x) * denom, bounds); } else { + //System.out.println("Z biggest"); // Z is the biggest, so Z0 = 0.0 if (Math.abs(denomXY) < MINIMUM_RESOLUTION_SQUARED) { - //System.err.println(" Denominator is zero: no intersection"); + //System.out.println(" Denominator is zero: no intersection"); return; } final double denom = 1.0 / denomXY; @@ -1178,6 +1182,10 @@ public class Plane extends Vector { if (point2Valid) { boundsInfo.addPoint(new GeoPoint(point2X, point2Y, point2Z)); } + } else { + // If we can't intersect line with world, then it's outside the world, so + // we have to assume everything is included. + boundsInfo.noBound(planetModel); } } @@ -1351,8 +1359,6 @@ public class Plane extends Vector { // m * [- 2*A*ab^2*r + 2*A^2*ab^2*r*q + 2*B^2*ab^2*r*q + 2*C^2*c^2*r*q] + // [ab^2 - 2*A*ab^2*q + A^2*ab^2*q^2 + B^2*ab^2*q^2 + C^2*c^2*q^2] = 0 - //System.err.println(" computing X bound"); - // Useful subexpressions for this bound final double q = A*abSquared*k; final double qSquared = q * q; @@ -1392,29 +1398,33 @@ public class Plane extends Vector { assert Math.abs(a * m1 * m1 + b * m1 + c) < MINIMUM_RESOLUTION; final double m2 = (-b - sqrtResult) * commonDenom; assert Math.abs(a * m2 * m2 + b * m2 + c) < MINIMUM_RESOLUTION; - final double l1 = r * m1 + q; - final double l2 = r * m2 + q; - // x = ((1 - l*A) * ab^2 ) / (2 * m) - // y = (-l*B * ab^2) / ( 2 * m) - // z = (-l*C * c^2)/ (2 * m) - final double denom1 = 0.5 / m1; - final double denom2 = 0.5 / m2; - final GeoPoint thePoint1 = new GeoPoint((1.0-l1*A) * abSquared * denom1, -l1*B * abSquared * denom1, -l1*C * cSquared * denom1); - final GeoPoint thePoint2 = new GeoPoint((1.0-l2*A) * abSquared * denom2, -l2*B * abSquared * denom2, -l2*C * cSquared * denom2); - //Math is not quite accurate enough for this - //assert planetModel.pointOnSurface(thePoint1): "Point1: "+thePoint1+"; Planetmodel="+planetModel+"; A="+A+" B="+B+" C="+C+" D="+D+" planetfcn="+ - // (thePoint1.x*thePoint1.x*planetModel.inverseAb*planetModel.inverseAb + thePoint1.y*thePoint1.y*planetModel.inverseAb*planetModel.inverseAb + thePoint1.z*thePoint1.z*planetModel.inverseC*planetModel.inverseC); - //assert planetModel.pointOnSurface(thePoint2): "Point1: "+thePoint2+"; Planetmodel="+planetModel+"; A="+A+" B="+B+" C="+C+" D="+D+" planetfcn="+ - // (thePoint2.x*thePoint2.x*planetModel.inverseAb*planetModel.inverseAb + thePoint2.y*thePoint2.y*planetModel.inverseAb*planetModel.inverseAb + thePoint2.z*thePoint2.z*planetModel.inverseC*planetModel.inverseC); - //assert evaluateIsZero(thePoint1): "Evaluation of point1: "+evaluate(thePoint1); - //assert evaluateIsZero(thePoint2): "Evaluation of point2: "+evaluate(thePoint2); - addPoint(boundsInfo, bounds, thePoint1); - addPoint(boundsInfo, bounds, thePoint2); + if (Math.abs(m1) >= MINIMUM_RESOLUTION || Math.abs(m2) >= MINIMUM_RESOLUTION) { + final double l1 = r * m1 + q; + final double l2 = r * m2 + q; + // x = ((1 - l*A) * ab^2 ) / (2 * m) + // y = (-l*B * ab^2) / ( 2 * m) + // z = (-l*C * c^2)/ (2 * m) + final double denom1 = 0.5 / m1; + final double denom2 = 0.5 / m2; + final GeoPoint thePoint1 = new GeoPoint((1.0-l1*A) * abSquared * denom1, -l1*B * abSquared * denom1, -l1*C * cSquared * denom1); + final GeoPoint thePoint2 = new GeoPoint((1.0-l2*A) * abSquared * denom2, -l2*B * abSquared * denom2, -l2*C * cSquared * denom2); + //Math is not quite accurate enough for this + //assert planetModel.pointOnSurface(thePoint1): "Point1: "+thePoint1+"; Planetmodel="+planetModel+"; A="+A+" B="+B+" C="+C+" D="+D+" planetfcn="+ + // (thePoint1.x*thePoint1.x*planetModel.inverseAb*planetModel.inverseAb + thePoint1.y*thePoint1.y*planetModel.inverseAb*planetModel.inverseAb + thePoint1.z*thePoint1.z*planetModel.inverseC*planetModel.inverseC); + //assert planetModel.pointOnSurface(thePoint2): "Point1: "+thePoint2+"; Planetmodel="+planetModel+"; A="+A+" B="+B+" C="+C+" D="+D+" planetfcn="+ + // (thePoint2.x*thePoint2.x*planetModel.inverseAb*planetModel.inverseAb + thePoint2.y*thePoint2.y*planetModel.inverseAb*planetModel.inverseAb + thePoint2.z*thePoint2.z*planetModel.inverseC*planetModel.inverseC); + //assert evaluateIsZero(thePoint1): "Evaluation of point1: "+evaluate(thePoint1); + //assert evaluateIsZero(thePoint2): "Evaluation of point2: "+evaluate(thePoint2); + addPoint(boundsInfo, bounds, thePoint1); + addPoint(boundsInfo, bounds, thePoint2); + } else { + // This is a plane of the form A=n B=0 C=0. We can set a bound only by noting the D value. + boundsInfo.addXValue(-D/A); + } } else { // No solutions } } else if (Math.abs(b) > MINIMUM_RESOLUTION_SQUARED) { - //System.err.println("Not x quadratic"); // a = 0, so m = - c / b final double m = -c / b; final double l = r * m + q; @@ -1561,24 +1571,29 @@ public class Plane extends Vector { assert Math.abs(a * m1 * m1 + b * m1 + c) < MINIMUM_RESOLUTION; final double m2 = (-b - sqrtResult) * commonDenom; assert Math.abs(a * m2 * m2 + b * m2 + c) < MINIMUM_RESOLUTION; - final double l1 = r * m1 + q; - final double l2 = r * m2 + q; - // x = (-l*A * ab^2 ) / (2 * m) - // y = ((1.0-l*B) * ab^2) / ( 2 * m) - // z = (-l*C * c^2)/ (2 * m) - final double denom1 = 0.5 / m1; - final double denom2 = 0.5 / m2; - final GeoPoint thePoint1 = new GeoPoint(-l1*A * abSquared * denom1, (1.0-l1*B) * abSquared * denom1, -l1*C * cSquared * denom1); - final GeoPoint thePoint2 = new GeoPoint(-l2*A * abSquared * denom2, (1.0-l2*B) * abSquared * denom2, -l2*C * cSquared * denom2); - //Math is not quite accurate enough for this - //assert planetModel.pointOnSurface(thePoint1): "Point1: "+thePoint1+"; Planetmodel="+planetModel+"; A="+A+" B="+B+" C="+C+" D="+D+" planetfcn="+ - // (thePoint1.x*thePoint1.x*planetModel.inverseAb*planetModel.inverseAb + thePoint1.y*thePoint1.y*planetModel.inverseAb*planetModel.inverseAb + thePoint1.z*thePoint1.z*planetModel.inverseC*planetModel.inverseC); - //assert planetModel.pointOnSurface(thePoint2): "Point2: "+thePoint2+"; Planetmodel="+planetModel+"; A="+A+" B="+B+" C="+C+" D="+D+" planetfcn="+ - // (thePoint2.x*thePoint2.x*planetModel.inverseAb*planetModel.inverseAb + thePoint2.y*thePoint2.y*planetModel.inverseAb*planetModel.inverseAb + thePoint2.z*thePoint2.z*planetModel.inverseC*planetModel.inverseC); - //assert evaluateIsZero(thePoint1): "Evaluation of point1: "+evaluate(thePoint1); - //assert evaluateIsZero(thePoint2): "Evaluation of point2: "+evaluate(thePoint2); - addPoint(boundsInfo, bounds, thePoint1); - addPoint(boundsInfo, bounds, thePoint2); + if (Math.abs(m1) >= MINIMUM_RESOLUTION || Math.abs(m2) >= MINIMUM_RESOLUTION) { + final double l1 = r * m1 + q; + final double l2 = r * m2 + q; + // x = (-l*A * ab^2 ) / (2 * m) + // y = ((1.0-l*B) * ab^2) / ( 2 * m) + // z = (-l*C * c^2)/ (2 * m) + final double denom1 = 0.5 / m1; + final double denom2 = 0.5 / m2; + final GeoPoint thePoint1 = new GeoPoint(-l1*A * abSquared * denom1, (1.0-l1*B) * abSquared * denom1, -l1*C * cSquared * denom1); + final GeoPoint thePoint2 = new GeoPoint(-l2*A * abSquared * denom2, (1.0-l2*B) * abSquared * denom2, -l2*C * cSquared * denom2); + //Math is not quite accurate enough for this + //assert planetModel.pointOnSurface(thePoint1): "Point1: "+thePoint1+"; Planetmodel="+planetModel+"; A="+A+" B="+B+" C="+C+" D="+D+" planetfcn="+ + // (thePoint1.x*thePoint1.x*planetModel.inverseAb*planetModel.inverseAb + thePoint1.y*thePoint1.y*planetModel.inverseAb*planetModel.inverseAb + thePoint1.z*thePoint1.z*planetModel.inverseC*planetModel.inverseC); + //assert planetModel.pointOnSurface(thePoint2): "Point2: "+thePoint2+"; Planetmodel="+planetModel+"; A="+A+" B="+B+" C="+C+" D="+D+" planetfcn="+ + // (thePoint2.x*thePoint2.x*planetModel.inverseAb*planetModel.inverseAb + thePoint2.y*thePoint2.y*planetModel.inverseAb*planetModel.inverseAb + thePoint2.z*thePoint2.z*planetModel.inverseC*planetModel.inverseC); + //assert evaluateIsZero(thePoint1): "Evaluation of point1: "+evaluate(thePoint1); + //assert evaluateIsZero(thePoint2): "Evaluation of point2: "+evaluate(thePoint2); + addPoint(boundsInfo, bounds, thePoint1); + addPoint(boundsInfo, bounds, thePoint2); + } else { + // This is a plane of the form A=0 B=n C=0. We can set a bound only by noting the D value. + boundsInfo.addYValue(-D/B); + } } else { // No solutions } diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZBounds.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZBounds.java index 85659ab6cc8..919b72e0294 100644 --- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZBounds.java +++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZBounds.java @@ -292,6 +292,17 @@ public class XYZBounds implements Bounds { return this; } + @Override + public Bounds noBound(final PlanetModel planetModel) { + minX = planetModel.getMinimumXValue(); + maxX = planetModel.getMaximumXValue(); + minY = planetModel.getMinimumYValue(); + maxY = planetModel.getMaximumYValue(); + minZ = planetModel.getMinimumZValue(); + maxZ = planetModel.getMaximumZValue(); + return this; + } + @Override public String toString() { return "XYZBounds: [xmin="+minX+" xmax="+maxX+" ymin="+minY+" ymax="+maxY+" zmin="+minZ+" zmax="+maxZ+"]"; diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java index bb98145d631..d9baf612e2b 100644 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/TestGeo3DPoint.java @@ -87,8 +87,9 @@ public class TestGeo3DPoint extends LuceneTestCase { private static Codec getCodec() { if (Codec.getDefault().getName().equals("Lucene62")) { int maxPointsInLeafNode = TestUtil.nextInt(random(), 16, 2048); + double maxMBSortInHeap = 3.0 + (3*random().nextDouble()); if (VERBOSE) { - System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode); + System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap); } return new FilterCodec("Lucene62", Codec.getDefault()) { @@ -97,7 +98,7 @@ public class TestGeo3DPoint extends LuceneTestCase { return new PointsFormat() { @Override public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException { - return new Lucene60PointsWriter(writeState, maxPointsInLeafNode); + return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap); } @Override diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoBBoxTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoBBoxTest.java index 993c79f759a..7530c7406e9 100755 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoBBoxTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoBBoxTest.java @@ -372,4 +372,19 @@ public class GeoBBoxTest { assertTrue(box.isWithin(point)?solid.isWithin(point):true); } + @Test + public void testFailureCase2() { + //final GeoPoint point = new GeoPoint(-0.7375647084975573, -2.3309121299774915E-10, 0.6746626163258577); + final GeoPoint point = new GeoPoint(-0.737564708579924, -9.032562595264542E-17, 0.6746626165197899); + final GeoBBox box = new GeoRectangle(PlanetModel.WGS84, 0.7988584710911523, 0.25383311815493353, -1.2236144735575564E-12, 7.356011300929654E-49); + final XYZBounds bounds = new XYZBounds(); + box.getBounds(bounds); + final XYZSolid solid = XYZSolidFactory.makeXYZSolid(PlanetModel.WGS84, bounds.getMinimumX(), bounds.getMaximumX(), bounds.getMinimumY(), bounds.getMaximumY(), bounds.getMinimumZ(), bounds.getMaximumZ()); + + //System.out.println("Is within Y value? "+(point.y >= bounds.getMinimumY() && point.y <= bounds.getMaximumY())); + //System.out.println("Shape = "+box+" is within? "+box.isWithin(point)); + //System.out.println("XYZBounds = "+bounds+" is within? "+solid.isWithin(point)+" solid="+solid); + assertTrue(box.isWithin(point) == solid.isWithin(point)); + } + } diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoCircleTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoCircleTest.java index 6f9a86b5d23..0190cde7f02 100755 --- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoCircleTest.java +++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoCircleTest.java @@ -405,4 +405,18 @@ public class GeoCircleTest extends LuceneTestCase { assertTrue(solid.isWithin(gp)); } + @Test + public void testBoundsFailureCase2() { + final GeoCircle gc = GeoCircleFactory.makeGeoCircle(PlanetModel.WGS84, -2.7574435614238194E-13, 0.0, 1.5887859182593391); + final GeoPoint gp = new GeoPoint(PlanetModel.WGS84, 0.7980359504429014, 1.5964981068121482); + final XYZBounds bounds = new XYZBounds(); + gc.getBounds(bounds); + System.out.println("Bounds = "+bounds); + System.out.println("Point = "+gp); + final XYZSolid solid = XYZSolidFactory.makeXYZSolid(PlanetModel.WGS84, bounds.getMinimumX(), bounds.getMaximumX(), bounds.getMinimumY(), bounds.getMaximumY(), bounds.getMinimumZ(), bounds.getMaximumZ()); + + assert gc.isWithin(gp)?solid.isWithin(gp):true; + + } + } diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java index c3c672b6475..c6f5485202a 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointsFormat.java @@ -126,6 +126,7 @@ public final class AssertingPointsFormat extends PointsFormat { assert false: "point values are out of order"; } System.arraycopy(packedValue, 0, lastDocValue, 0, bytesPerDim); + lastDocID = docID; } in.visit(docID, packedValue); } @@ -254,11 +255,11 @@ public final class AssertingPointsFormat extends PointsFormat { } @Override - public void writeField(FieldInfo fieldInfo, PointsReader values, double maxMBSortInHeap) throws IOException { + public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException { if (fieldInfo.getPointDimensionCount() == 0) { throw new IllegalArgumentException("writing field=\"" + fieldInfo.name + "\" but pointDimensionalCount is 0"); } - in.writeField(fieldInfo, values, maxMBSortInHeap); + in.writeField(fieldInfo, values); } @Override diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java index ffd9a8c4eab..fd2260be6a5 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointsFormat.java @@ -56,11 +56,11 @@ class CrankyPointsFormat extends PointsFormat { } @Override - public void writeField(FieldInfo fieldInfo, PointsReader values, double maxMBSortInHeap) throws IOException { + public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException { if (random.nextInt(100) == 0) { throw new IOException("Fake IOException"); } - delegate.writeField(fieldInfo, values, maxMBSortInHeap); + delegate.writeField(fieldInfo, values); } @Override diff --git a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java index 926132fa061..275c1864857 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/geo/BaseGeoPointTestCase.java @@ -67,6 +67,7 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SloppyMath; import org.apache.lucene.util.TestUtil; +import org.apache.lucene.util.bkd.BKDWriter; /** * Abstract class to do basic tests for a geospatial impl (high level @@ -1247,7 +1248,7 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase { return new PointsFormat() { @Override public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException { - return new Lucene60PointsWriter(writeState, pointsInLeaf); + return new Lucene60PointsWriter(writeState, pointsInLeaf, BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP); } @Override diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java index 6e8f7bcb14a..127549ff065 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java @@ -92,6 +92,7 @@ public class RandomCodec extends AssertingCodec { // which is less effective for testing. // TODO: improve how we randomize this... private final int maxPointsInLeafNode; + private final double maxMBSortInHeap; private final int bkdSplitRandomSeed; @Override @@ -102,9 +103,9 @@ public class RandomCodec extends AssertingCodec { // Randomize how BKDWriter chooses its splis: - return new Lucene60PointsWriter(writeState, maxPointsInLeafNode) { + return new Lucene60PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap) { @Override - public void writeField(FieldInfo fieldInfo, PointsReader values, double maxMBSortInHeap) throws IOException { + public void writeField(FieldInfo fieldInfo, PointsReader values) throws IOException { boolean singleValuePerDoc = values.size(fieldInfo.name) == values.getDocCount(fieldInfo.name); @@ -184,6 +185,7 @@ public class RandomCodec extends AssertingCodec { int lowFreqCutoff = TestUtil.nextInt(random, 2, 100); maxPointsInLeafNode = TestUtil.nextInt(random, 16, 2048); + maxMBSortInHeap = 5.0 + (3*random.nextDouble()); bkdSplitRandomSeed = random.nextInt(); add(avoidCodecs, @@ -251,7 +253,8 @@ public class RandomCodec extends AssertingCodec { public String toString() { return super.toString() + ": " + previousMappings.toString() + ", docValues:" + previousDVMappings.toString() + - ", maxPointsInLeafNode=" + maxPointsInLeafNode; + ", maxPointsInLeafNode=" + maxPointsInLeafNode + + ", maxMBSortInHeap=" + maxMBSortInHeap; } /** Just like {@link BKDWriter} except it evilly picks random ways to split cells on diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java index e78968d452a..1ff9470fb97 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java @@ -771,7 +771,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { } ii = new SlowOpeningMockIndexInputWrapper(this, name, delegateInput); } else { - ii = new MockIndexInputWrapper(this, name, delegateInput); + ii = new MockIndexInputWrapper(this, name, delegateInput, null); } addFileHandle(ii, name, Handle.Input); return ii; diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexInputWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexInputWrapper.java index f62d67b9392..f68e18cd8ab 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexInputWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexInputWrapper.java @@ -30,12 +30,19 @@ public class MockIndexInputWrapper extends IndexInput { private MockDirectoryWrapper dir; final String name; private IndexInput delegate; - private boolean isClone; - private boolean closed; + private volatile boolean closed; - /** Construct an empty output buffer. */ - public MockIndexInputWrapper(MockDirectoryWrapper dir, String name, IndexInput delegate) { + // Which MockIndexInputWrapper we were cloned from, or null if we are not a clone: + private final MockIndexInputWrapper parent; + + /** Sole constructor */ + public MockIndexInputWrapper(MockDirectoryWrapper dir, String name, IndexInput delegate, MockIndexInputWrapper parent) { super("MockIndexInputWrapper(name=" + name + " delegate=" + delegate + ")"); + + // If we are a clone then our parent better not be a clone! + assert parent == null || parent.parent == null; + + this.parent = parent; this.name = name; this.dir = dir; this.delegate = delegate; @@ -54,7 +61,7 @@ public class MockIndexInputWrapper extends IndexInput { // remove the conditional check so we also track that // all clones get closed: assert delegate != null; - if (!isClone) { + if (parent == null) { dir.removeIndexInput(this, name); } dir.maybeThrowDeterministicException(); @@ -62,9 +69,13 @@ public class MockIndexInputWrapper extends IndexInput { } private void ensureOpen() { + // TODO: not great this is a volatile read (closed) ... we should deploy heavy JVM voodoo like SwitchPoint to avoid this if (closed) { throw new RuntimeException("Abusing closed IndexInput!"); } + if (parent != null && parent.closed) { + throw new RuntimeException("Abusing clone of a closed IndexInput!"); + } } @Override @@ -75,8 +86,7 @@ public class MockIndexInputWrapper extends IndexInput { } dir.inputCloneCount.incrementAndGet(); IndexInput iiclone = delegate.clone(); - MockIndexInputWrapper clone = new MockIndexInputWrapper(dir, name, iiclone); - clone.isClone = true; + MockIndexInputWrapper clone = new MockIndexInputWrapper(dir, name, iiclone, parent != null ? parent : this); // Pending resolution on LUCENE-686 we may want to // uncomment this code so that we also track that all // clones get closed: @@ -102,8 +112,7 @@ public class MockIndexInputWrapper extends IndexInput { } dir.inputCloneCount.incrementAndGet(); IndexInput slice = delegate.slice(sliceDescription, offset, length); - MockIndexInputWrapper clone = new MockIndexInputWrapper(dir, sliceDescription, slice); - clone.isClone = true; + MockIndexInputWrapper clone = new MockIndexInputWrapper(dir, sliceDescription, slice, parent != null ? parent : this); return clone; } diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/SlowClosingMockIndexInputWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/SlowClosingMockIndexInputWrapper.java index 2be2e2792b3..e6c3857164e 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/SlowClosingMockIndexInputWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/SlowClosingMockIndexInputWrapper.java @@ -30,7 +30,7 @@ class SlowClosingMockIndexInputWrapper extends MockIndexInputWrapper { public SlowClosingMockIndexInputWrapper(MockDirectoryWrapper dir, String name, IndexInput delegate) { - super(dir, name, delegate); + super(dir, name, delegate, null); } @Override diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/SlowOpeningMockIndexInputWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/SlowOpeningMockIndexInputWrapper.java index 4cc2b19665c..1e95451ec3d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/SlowOpeningMockIndexInputWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/SlowOpeningMockIndexInputWrapper.java @@ -28,7 +28,7 @@ class SlowOpeningMockIndexInputWrapper extends MockIndexInputWrapper { public SlowOpeningMockIndexInputWrapper(MockDirectoryWrapper dir, String name, IndexInput delegate) throws IOException { - super(dir, name, delegate); + super(dir, name, delegate, null); try { Thread.sleep(50); } catch (InterruptedException ie) { diff --git a/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java b/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java index ae453e5155e..0605ebd3d8a 100644 --- a/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java +++ b/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java @@ -171,4 +171,40 @@ public class TestMockDirectoryWrapper extends BaseDirectoryTestCase { assertTrue("MockDirectoryWrapper on dir=" + dir + " failed to corrupt an unsync'd file", changed); } + + public void testAbuseClosedIndexInput() throws Exception { + MockDirectoryWrapper dir = newMockDirectory(); + IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT); + out.writeByte((byte) 42); + out.close(); + final IndexInput in = dir.openInput("foo", IOContext.DEFAULT); + in.close(); + expectThrows(RuntimeException.class, in::readByte); + dir.close(); + } + + public void testAbuseCloneAfterParentClosed() throws Exception { + MockDirectoryWrapper dir = newMockDirectory(); + IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT); + out.writeByte((byte) 42); + out.close(); + IndexInput in = dir.openInput("foo", IOContext.DEFAULT); + final IndexInput clone = in.clone(); + in.close(); + expectThrows(RuntimeException.class, clone::readByte); + dir.close(); + } + + public void testAbuseCloneOfCloneAfterParentClosed() throws Exception { + MockDirectoryWrapper dir = newMockDirectory(); + IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT); + out.writeByte((byte) 42); + out.close(); + IndexInput in = dir.openInput("foo", IOContext.DEFAULT); + IndexInput clone1 = in.clone(); + IndexInput clone2 = clone1.clone(); + in.close(); + expectThrows(RuntimeException.class, clone2::readByte); + dir.close(); + } } diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 889611f6b9a..518f63a10d9 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -100,6 +100,25 @@ New Features * SOLR-9275: XML QueryParser support (defType=xmlparser) now extensible via configuration. (Christine Poerschke) +* SOLR-9200: Add Delegation Token Support to Solr. + (Gregory Chanan) + +* SOLR-9038: Solr core snapshots: The current commit can be snapshotted which retains the commit and associates it with + a name. The core admin API can create snapshots, list them, and delete them. Snapshot names can be referenced in + doing a core backup, and in replication. Snapshot metadata is stored in a new snapshot_metadata/ dir. + (Hrishikesh Gadre via David Smiley) + +* SOLR-9279: New boolean comparison function queries comparing numeric arguments: gt, gte, lt, lte, eq + (Doug Turnbull, David Smiley) + +* SOLR-9324: Support Secure Impersonation / Proxy User for solr authentication + (Gregory Chanan) + +* SOLR-9252: Feature selection and logistic regression on text (Cao Manh Dat, Joel Bernstein) + +* SOLR-6465: CDCR: fall back to whole-index replication when tlogs are insufficient. + (Noble Paul, Renaud Delbru, shalin) + * SOLR-9320: A REPLACENODE command to decommission an existing node with another new node (noble, Nitin Sharma, Varun Thacker) @@ -170,6 +189,19 @@ Bug Fixes * SOLR-9339: NPE in CloudSolrClient when the response is null (noble) +* SOLR-8596: Web UI doesn't correctly generate queries which include local parameters (Alexandre Rafalovitch, janhoy) + +* SOLR-8645: managed-schema is now syntax highlighted in cloud->Tree view (Alexandre Rafalovitch via janhoy) + +* SOLR-8379: UI Cloud->Tree view now shows .txt files correctly (Alexandre Rafalovitch via janhoy) + +* SOLR-9003: New Admin UI's Dataimport screen now correctly displays DIH Debug output (Alexandre Rafalovitch) + +* SOLR-9308: Fix distributed RTG to forward request params, fixes fq and non-default fl params (hossman) + +* SOLR-9179: NPE in IndexSchema using IBM JDK (noble, Colvin Cowie) + +* SOLR-9397: Config API does not support adding caches (noble) Optimizations ---------------------- @@ -179,6 +211,13 @@ Optimizations * SOLR-9264: Optimize ZkController.publishAndWaitForDownStates to not read all collection states and watch relevant collections instead. (Hrishikesh Gadre, shalin) +* SOLR-9335: Solr cache/search/update stats counters now use LongAdder which are supposed to have higher throughput + under high contention. (Varun Thacker) + +* SOLR-9350: JSON Facets: method="stream" will no longer always uses & populates the filter cache, likely + flushing it. 'cacheDf' can be configured to set a doc frequency threshold, now defaulting to 1/16th doc count. + Using -1 Disables use of the cache. (David Smiley, yonik) + Other Changes ---------------------- @@ -202,6 +241,25 @@ Other Changes * SOLR-9163: Sync up basic_configs and data_driven_schema_configs, removing almost all differences except what is required for schemaless. (yonik) +* SOLR-9340: Change ZooKeeper disconnect and session expiry related logging from INFO to WARN to + make debugging easier (Varun Thacker) + +* SOLR-9358: [AngularUI] In Cloud->Tree file view area, collapse metadata by default (janhoy) + +* SOLR-9256: asserting hasNext() contract in JdbcDataSource in DataImportHandler (Kristine Jetzke via Mikhai Khludnev) + +* SOLR-9209: extracting JdbcDataSource.createResultSetIterator() for extension (Kristine Jetzke via Mikhai Khludnev) + +* SOLR-9353: Factor out ReRankQParserPlugin.ReRankQueryRescorer private class. (Christine Poerschke) + +* SOLR-9392: Fixed CDCR Test failures which were due to leaked resources. (shalin) + +* SOLR-9385: Add QParser.getParser(String,SolrQueryRequest) variant. (Christine Poerschke) + +* SOLR-9367: Improved TestInjection's randomization logic to use LuceneTestCase.random() (hossman) + +* SOLR-9331: Remove ReRankQuery's length constructor argument and member. (Christine Poerschke) + ================== 6.1.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java index 03392f0ef1a..d8828a6ed19 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java @@ -604,7 +604,7 @@ public class FacetingAccumulator extends BasicAccumulator implements FacetValueA QueryFacetAccumulator qAcc = new QueryFacetAccumulator(this,qfr.getName(),query); final Query q; try { - q = QParser.getParser(query, null, queryRequest).getQuery(); + q = QParser.getParser(query, queryRequest).getQuery(); } catch( SyntaxError e ){ throw new SolrException(ErrorCode.BAD_REQUEST,"Invalid query '"+query+"'",e); } diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java index 09ad775a4ec..ce8671a0f51 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/JdbcDataSource.java @@ -280,10 +280,14 @@ public class JdbcDataSource extends resultSetIterator.close(); resultSetIterator = null; } - resultSetIterator = new ResultSetIterator(query); + resultSetIterator = createResultSetIterator(query); return resultSetIterator.getIterator(); } + protected ResultSetIterator createResultSetIterator(String query) { + return new ResultSetIterator(query); + } + private void logError(String msg, Exception e) { LOG.warn(msg, e); } diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java index b6d05c441f1..7853ad154f6 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestJdbcDataSource.java @@ -510,6 +510,45 @@ public class TestJdbcDataSource extends AbstractDataImportHandlerTestCase { DriverManager.deregisterDriver(driver); } } + + + @Test + public void testEmptyResultSet() throws Exception { + MockInitialContextFactory.bind("java:comp/env/jdbc/JndiDB", dataSource); + + props.put(JdbcDataSource.JNDI_NAME, "java:comp/env/jdbc/JndiDB"); + EasyMock.expect(dataSource.getConnection()).andReturn(connection); + + jdbcDataSource.init(context, props); + + connection.setAutoCommit(false); + + Statement statement = mockControl.createMock(Statement.class); + EasyMock.expect(connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) + .andReturn(statement); + statement.setFetchSize(500); + statement.setMaxRows(0); + EasyMock.expect(statement.execute("query")).andReturn(true); + ResultSet resultSet = mockControl.createMock(ResultSet.class); + EasyMock.expect(statement.getResultSet()).andReturn(resultSet); + ResultSetMetaData metaData = mockControl.createMock(ResultSetMetaData.class); + EasyMock.expect(resultSet.getMetaData()).andReturn(metaData); + EasyMock.expect(metaData.getColumnCount()).andReturn(0); + EasyMock.expect(resultSet.next()).andReturn(false); + resultSet.close(); + EasyMock.expect(statement.getMoreResults()).andReturn(false); + EasyMock.expect(statement.getUpdateCount()).andReturn(-1); + statement.close(); + + mockControl.replay(); + + Iterator> resultSetIterator = jdbcDataSource.getData("query"); + resultSetIterator.hasNext(); + resultSetIterator.hasNext(); + + mockControl.verify(); + } + @Test @Ignore("Needs a Mock database server to work") public void testBasic() throws Exception { diff --git a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ZooKeeperInspector.java b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ZooKeeperInspector.java index 9b86dcda97a..76928aa17d7 100644 --- a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ZooKeeperInspector.java +++ b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/ZooKeeperInspector.java @@ -16,6 +16,15 @@ */ package org.apache.solr.hadoop; +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + import com.google.common.io.Files; import org.apache.commons.io.FileUtils; import org.apache.solr.cloud.ZkController; @@ -35,15 +44,6 @@ import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - /** * Extracts SolrCloud information from ZooKeeper. */ @@ -78,8 +78,7 @@ final class ZooKeeperInspector { } SolrZkClient zkClient = getZkClient(zkHost); - try { - ZkStateReader zkStateReader = new ZkStateReader(zkClient); + try (ZkStateReader zkStateReader = new ZkStateReader(zkClient)) { try { // first check for alias collection = checkForAlias(zkClient, collection); diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml index 5dad49bad0a..08272ad7448 100644 --- a/solr/core/ivy.xml +++ b/solr/core/ivy.xml @@ -134,6 +134,10 @@ + + + + diff --git a/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java b/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java index 207c0e5d2a0..3616d4e545b 100644 --- a/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java +++ b/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java @@ -15,21 +15,26 @@ * limitations under the License. */ package org.apache.solr.core; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.Directory; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.update.SolrIndexWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - /** * A wrapper for an IndexDeletionPolicy instance. *

@@ -52,9 +57,11 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy { private final Map reserves = new ConcurrentHashMap<>(); private volatile IndexCommit latestCommit; private final ConcurrentHashMap savedCommits = new ConcurrentHashMap<>(); + private final SolrSnapshotMetaDataManager snapshotMgr; - public IndexDeletionPolicyWrapper(IndexDeletionPolicy deletionPolicy) { + public IndexDeletionPolicyWrapper(IndexDeletionPolicy deletionPolicy, SolrSnapshotMetaDataManager snapshotMgr) { this.deletionPolicy = deletionPolicy; + this.snapshotMgr = snapshotMgr; } /** @@ -134,7 +141,6 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy { } } - /** * Internal use for Lucene... do not explicitly call. */ @@ -185,7 +191,8 @@ public final class IndexDeletionPolicyWrapper extends IndexDeletionPolicy { Long gen = delegate.getGeneration(); Long reserve = reserves.get(gen); if (reserve != null && System.nanoTime() < reserve) return; - if(savedCommits.containsKey(gen)) return; + if (savedCommits.containsKey(gen)) return; + if (snapshotMgr.isSnapshotted(gen)) return; delegate.delete(); } diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index eb3aa5fc7f1..653c612fe65 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -28,7 +28,17 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -49,6 +59,7 @@ import org.apache.solr.schema.IndexSchemaFactory; import org.apache.solr.search.CacheConfig; import org.apache.solr.search.FastLRUCache; import org.apache.solr.search.QParserPlugin; +import org.apache.solr.search.SolrCache; import org.apache.solr.search.ValueSourceParser; import org.apache.solr.search.stats.StatsCache; import org.apache.solr.servlet.SolrRequestParsers; @@ -91,7 +102,7 @@ public class SolrConfig extends Config implements MapSerializable { public static final String DEFAULT_CONF_FILE = "solrconfig.xml"; private RequestParams requestParams; - public static enum PluginOpts { + public enum PluginOpts { MULTI_OK, REQUIRE_NAME, REQUIRE_NAME_IN_OVERLAY, @@ -254,7 +265,6 @@ public class SolrConfig extends Config implements MapSerializable { dataDir = get("dataDir", null); if (dataDir != null && dataDir.length() == 0) dataDir = null; - userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache"); org.apache.solr.search.SolrIndexSearcher.initRegenerators(this); @@ -276,6 +286,16 @@ public class SolrConfig extends Config implements MapSerializable { maxWarmingSearchers = getInt("query/maxWarmingSearchers", Integer.MAX_VALUE); slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1); for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin); + + Map userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache"); + List caches = getPluginInfos(SolrCache.class.getName()); + if (!caches.isEmpty()) { + for (PluginInfo c : caches) { + userCacheConfigs.put(c.name, CacheConfig.getConfig(this, "cache", c.attributes, null)); + } + } + this.userCacheConfigs = Collections.unmodifiableMap(userCacheConfigs); + updateHandlerInfo = loadUpdatehandlerInfo(); multipartUploadLimitKB = getInt( @@ -317,6 +337,7 @@ public class SolrConfig extends Config implements MapSerializable { .add(new SolrPluginInfo(TransformerFactory.class, "transformer", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK)) .add(new SolrPluginInfo(SearchComponent.class, "searchComponent", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK)) .add(new SolrPluginInfo(UpdateRequestProcessorFactory.class, "updateProcessor", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK)) + .add(new SolrPluginInfo(SolrCache.class, "cache", REQUIRE_NAME, REQUIRE_CLASS, MULTI_OK)) // TODO: WTF is up with queryConverter??? // it apparently *only* works as a singleton? - SOLR-4304 // and even then -- only if there is a single SpellCheckComponent @@ -457,7 +478,7 @@ public class SolrConfig extends Config implements MapSerializable { public final CacheConfig queryResultCacheConfig; public final CacheConfig documentCacheConfig; public final CacheConfig fieldValueCacheConfig; - public final CacheConfig[] userCacheConfigs; + public final Map userCacheConfigs; // SolrIndexSearcher - more... public final boolean useFilterForSortedQuery; public final int queryResultWindowSize; diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index faef1c8a4fd..2704e4ac868 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -81,6 +81,7 @@ import org.apache.solr.common.util.ObjectReleaseTracker; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.Utils; import org.apache.solr.core.DirectoryFactory.DirContext; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.handler.IndexFetcher; import org.apache.solr.handler.ReplicationHandler; import org.apache.solr.handler.RequestHandlerBase; @@ -184,6 +185,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable { private final Map updateProcessorChains; private final Map infoRegistry; private final IndexDeletionPolicyWrapper solrDelPolicy; + private final SolrSnapshotMetaDataManager snapshotMgr; private final DirectoryFactory directoryFactory; private IndexReaderFactory indexReaderFactory; private final Codec codec; @@ -414,7 +416,19 @@ public final class SolrCore implements SolrInfoMBean, Closeable { } else { delPolicy = new SolrDeletionPolicy(); } - return new IndexDeletionPolicyWrapper(delPolicy); + + return new IndexDeletionPolicyWrapper(delPolicy, snapshotMgr); + } + + private SolrSnapshotMetaDataManager initSnapshotMetaDataManager() { + try { + String dirName = getDataDir() + SolrSnapshotMetaDataManager.SNAPSHOT_METADATA_DIR + "/"; + Directory snapshotDir = directoryFactory.get(dirName, DirContext.DEFAULT, + getSolrConfig().indexConfig.lockType); + return new SolrSnapshotMetaDataManager(this, snapshotDir); + } catch (IOException e) { + throw new IllegalStateException(e); + } } private void initListeners() { @@ -739,6 +753,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable { initListeners(); + this.snapshotMgr = initSnapshotMetaDataManager(); this.solrDelPolicy = initDeletionPolicy(delPolicy); this.codec = initCodec(solrConfig, this.schema); @@ -1242,6 +1257,17 @@ public final class SolrCore implements SolrInfoMBean, Closeable { } } + // Close the snapshots meta-data directory. + Directory snapshotsDir = snapshotMgr.getSnapshotsDir(); + try { + this.directoryFactory.release(snapshotsDir); + } catch (Throwable e) { + SolrException.log(log,e); + if (e instanceof Error) { + throw (Error) e; + } + } + if (coreStateClosed) { try { @@ -2343,6 +2369,14 @@ public final class SolrCore implements SolrInfoMBean, Closeable { return solrDelPolicy; } + /** + * @return A reference of {@linkplain SolrSnapshotMetaDataManager} + * managing the persistent snapshots for this Solr core. + */ + public SolrSnapshotMetaDataManager getSnapshotMetaDataManager() { + return snapshotMgr; + } + public ReentrantLock getRuleExpiryLock() { return ruleExpiryLock; } diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java index 4eb77908c5a..86c411059d8 100644 --- a/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java +++ b/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java @@ -32,6 +32,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.NoLockFactory; import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.util.Constants; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.DirectoryFactory; @@ -59,10 +60,22 @@ public class LocalFileSystemRepository implements BackupRepository { @Override public URI createURI(String... pathComponents) { Preconditions.checkArgument(pathComponents.length > 0); - Path result = Paths.get(pathComponents[0]); + + String basePath = Preconditions.checkNotNull(pathComponents[0]); + // Note the URI.getPath() invocation on Windows platform generates an invalid URI. + // Refer to http://stackoverflow.com/questions/9834776/java-nio-file-path-issue + // Since the caller may have used this method to generate the string representation + // for the pathComponents, we implement a work-around specifically for Windows platform + // to remove the leading '/' character. + if (Constants.WINDOWS) { + basePath = basePath.replaceFirst("^/(.:/)", "$1"); + } + + Path result = Paths.get(basePath); for (int i = 1; i < pathComponents.length; i++) { result = result.resolve(pathComponents[i]); } + return result.toUri(); } diff --git a/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotManager.java b/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotManager.java new file mode 100644 index 00000000000..95df3fface2 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotManager.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core.snapshots; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.store.Directory; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class provides functionality required to handle the data files corresponding to Solr snapshots. + */ +public class SolrSnapshotManager { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * This method deletes index files of the {@linkplain IndexCommit} for the specified generation number. + * + * @param dir The index directory storing the snapshot. + * @param gen The generation number for the {@linkplain IndexCommit} + * @throws IOException in case of I/O errors. + */ + public static void deleteIndexFiles ( Directory dir, Collection snapshots, long gen ) throws IOException { + List commits = DirectoryReader.listCommits(dir); + Map refCounts = buildRefCounts(snapshots, commits); + for (IndexCommit ic : commits) { + if (ic.getGeneration() == gen) { + deleteIndexFiles(dir,refCounts, ic); + break; + } + } + } + + /** + * This method deletes all files not corresponding to a configured snapshot in the specified index directory. + * + * @param dir The index directory to search for. + * @throws IOException in case of I/O errors. + */ + public static void deleteNonSnapshotIndexFiles (Directory dir, Collection snapshots) throws IOException { + List commits = DirectoryReader.listCommits(dir); + Map refCounts = buildRefCounts(snapshots, commits); + Set snapshotGenNumbers = snapshots.stream() + .map(SnapshotMetaData::getGenerationNumber) + .collect(Collectors.toSet()); + for (IndexCommit ic : commits) { + if (!snapshotGenNumbers.contains(ic.getGeneration())) { + deleteIndexFiles(dir,refCounts, ic); + } + } + } + + /** + * This method computes reference count for the index files by taking into consideration + * (a) configured snapshots and (b) files sharing between two or more {@linkplain IndexCommit} instances. + * + * @param snapshots A collection of user configured snapshots + * @param commits A list of {@linkplain IndexCommit} instances + * @return A map containing reference count for each index file referred in one of the {@linkplain IndexCommit} instances. + * @throws IOException in case of I/O error. + */ + @VisibleForTesting + static Map buildRefCounts (Collection snapshots, List commits) throws IOException { + Map result = new HashMap<>(); + Map commitsByGen = commits.stream().collect( + Collectors.toMap(IndexCommit::getGeneration, Function.identity())); + + for(SnapshotMetaData md : snapshots) { + IndexCommit ic = commitsByGen.get(md.getGenerationNumber()); + if (ic != null) { + Collection fileNames = ic.getFileNames(); + for(String fileName : fileNames) { + int refCount = result.getOrDefault(fileName, 0); + result.put(fileName, refCount+1); + } + } + } + + return result; + } + + /** + * This method deletes the index files associated with specified indexCommit provided they + * are not referred by some other {@linkplain IndexCommit}. + * + * @param dir The index directory containing the {@linkplain IndexCommit} to be deleted. + * @param refCounts A map containing reference counts for each file associated with every {@linkplain IndexCommit} + * in the specified directory. + * @param indexCommit The {@linkplain IndexCommit} whose files need to be deleted. + * @throws IOException in case of I/O errors. + */ + private static void deleteIndexFiles ( Directory dir, Map refCounts, IndexCommit indexCommit ) throws IOException { + log.info("Deleting index files for index commit with generation {} in directory {}", indexCommit.getGeneration(), dir); + for (String fileName : indexCommit.getFileNames()) { + try { + // Ensure that a file being deleted is not referred by some other commit. + int ref = refCounts.getOrDefault(fileName, 0); + log.debug("Reference count for file {} is {}", fileName, ref); + if (ref == 0) { + dir.deleteFile(fileName); + } + } catch (IOException e) { + log.warn("Unable to delete file {} in directory {} due to exception {}", fileName, dir, e.getMessage()); + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotMetaDataManager.java b/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotMetaDataManager.java new file mode 100644 index 00000000000..26cbe215f70 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotMetaDataManager.java @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core.snapshots; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.google.common.base.Preconditions; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexDeletionPolicy; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.DirectoryFactory.DirContext; +import org.apache.solr.core.IndexDeletionPolicyWrapper; +import org.apache.solr.core.SolrCore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class is responsible to manage the persistent snapshots meta-data for the Solr indexes. The + * persistent snapshots are implemented by relying on Lucene {@linkplain IndexDeletionPolicy} + * abstraction to configure a specific {@linkplain IndexCommit} to be retained. The + * {@linkplain IndexDeletionPolicyWrapper} in Solr uses this class to create/delete the Solr index + * snapshots. + */ +public class SolrSnapshotMetaDataManager { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String SNAPSHOT_METADATA_DIR = "snapshot_metadata"; + + /** + * A class defining the meta-data for a specific snapshot. + */ + public static class SnapshotMetaData { + private String name; + private String indexDirPath; + private long generationNumber; + + public SnapshotMetaData(String name, String indexDirPath, long generationNumber) { + super(); + this.name = name; + this.indexDirPath = indexDirPath; + this.generationNumber = generationNumber; + } + + public String getName() { + return name; + } + + public String getIndexDirPath() { + return indexDirPath; + } + + public long getGenerationNumber() { + return generationNumber; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("SnapshotMetaData[name="); + builder.append(name); + builder.append(", indexDirPath="); + builder.append(indexDirPath); + builder.append(", generation="); + builder.append(generationNumber); + builder.append("]"); + return builder.toString(); + } + } + + /** Prefix used for the save file. */ + public static final String SNAPSHOTS_PREFIX = "snapshots_"; + private static final int VERSION_START = 0; + private static final int VERSION_CURRENT = VERSION_START; + private static final String CODEC_NAME = "solr-snapshots"; + + // The index writer which maintains the snapshots metadata + private long nextWriteGen; + + private final Directory dir; + + /** Used to map snapshot name to snapshot meta-data. */ + protected final Map nameToDetailsMapping = new LinkedHashMap<>(); + /** Used to figure out the *current* index data directory path */ + private final SolrCore solrCore; + + /** + * A constructor. + * + * @param dir The directory where the snapshot meta-data should be stored. Enables updating + * the existing meta-data. + * @throws IOException in case of errors. + */ + public SolrSnapshotMetaDataManager(SolrCore solrCore, Directory dir) throws IOException { + this(solrCore, dir, OpenMode.CREATE_OR_APPEND); + } + + /** + * A constructor. + * + * @param dir The directory where the snapshot meta-data is stored. + * @param mode CREATE If previous meta-data should be erased. + * APPEND If previous meta-data should be read and updated. + * CREATE_OR_APPEND Creates a new meta-data structure if one does not exist + * Updates the existing structure if one exists. + * @throws IOException in case of errors. + */ + public SolrSnapshotMetaDataManager(SolrCore solrCore, Directory dir, OpenMode mode) throws IOException { + this.solrCore = solrCore; + this.dir = dir; + + if (mode == OpenMode.CREATE) { + deleteSnapshotMetadataFiles(); + } + + loadFromSnapshotMetadataFile(); + + if (mode == OpenMode.APPEND && nextWriteGen == 0) { + throw new IllegalStateException("no snapshots stored in this directory"); + } + } + + /** + * @return The snapshot meta-data directory + */ + public Directory getSnapshotsDir() { + return dir; + } + + /** + * This method creates a new snapshot meta-data entry. + * + * @param name The name of the snapshot. + * @param indexDirPath The directory path where the index files are stored. + * @param gen The generation number for the {@linkplain IndexCommit} being snapshotted. + * @throws IOException in case of I/O errors. + */ + public synchronized void snapshot(String name, String indexDirPath, long gen) throws IOException { + Preconditions.checkNotNull(name); + + log.info("Creating the snapshot named {} for core {} associated with index commit with generation {} in directory {}" + , name, solrCore.getName(), gen, indexDirPath); + + if(nameToDetailsMapping.containsKey(name)) { + throw new SolrException(ErrorCode.BAD_REQUEST, "A snapshot with name " + name + " already exists"); + } + + SnapshotMetaData d = new SnapshotMetaData(name, indexDirPath, gen); + nameToDetailsMapping.put(name, d); + + boolean success = false; + try { + persist(); + success = true; + } finally { + if (!success) { + try { + release(name); + } catch (Exception e) { + // Suppress so we keep throwing original exception + } + } + } + } + + /** + * This method deletes a previously created snapshot (if any). + * + * @param name The name of the snapshot to be deleted. + * @return The snapshot meta-data if the snapshot with the snapshot name exists. + * @throws IOException in case of I/O error + */ + public synchronized Optional release(String name) throws IOException { + log.info("Deleting the snapshot named {} for core {}", name, solrCore.getName()); + SnapshotMetaData result = nameToDetailsMapping.remove(Preconditions.checkNotNull(name)); + if(result != null) { + boolean success = false; + try { + persist(); + success = true; + } finally { + if (!success) { + nameToDetailsMapping.put(name, result); + } + } + } + return Optional.ofNullable(result); + } + + /** + * This method returns if snapshot is created for the specified generation number in + * the *current* index directory. + * + * @param genNumber The generation number for the {@linkplain IndexCommit} to be checked. + * @return true if the snapshot is created. + * false otherwise. + */ + public synchronized boolean isSnapshotted(long genNumber) { + return !nameToDetailsMapping.isEmpty() && isSnapshotted(solrCore.getIndexDir(), genNumber); + } + + /** + * This method returns if snapshot is created for the specified generation number in + * the specified index directory. + * + * @param genNumber The generation number for the {@linkplain IndexCommit} to be checked. + * @return true if the snapshot is created. + * false otherwise. + */ + public synchronized boolean isSnapshotted(String indexDirPath, long genNumber) { + return !nameToDetailsMapping.isEmpty() + && nameToDetailsMapping.values().stream() + .anyMatch(entry -> entry.getIndexDirPath().equals(indexDirPath) && entry.getGenerationNumber() == genNumber); + } + + /** + * This method returns the snapshot meta-data for the specified name (if it exists). + * + * @param name The name of the snapshot + * @return The snapshot meta-data if exists. + */ + public synchronized Optional getSnapshotMetaData(String name) { + return Optional.ofNullable(nameToDetailsMapping.get(name)); + } + + /** + * @return A list of snapshots created so far. + */ + public synchronized List listSnapshots() { + // We create a copy for thread safety. + return new ArrayList<>(nameToDetailsMapping.keySet()); + } + + /** + * This method returns a list of snapshots created in a specified index directory. + * + * @param indexDirPath The index directory path. + * @return a list snapshots stored in the specified directory. + */ + public synchronized Collection listSnapshotsInIndexDir(String indexDirPath) { + return nameToDetailsMapping.values().stream() + .filter(entry -> indexDirPath.equals(entry.getIndexDirPath())) + .collect(Collectors.toList()); + } + + /** + * This method returns the {@linkplain IndexCommit} associated with the specified + * commitName. A snapshot with specified commitName must + * be created before invoking this method. + * + * @param commitName The name of persisted commit + * @return the {@linkplain IndexCommit} + * @throws IOException in case of I/O error. + */ + public Optional getIndexCommitByName(String commitName) throws IOException { + Optional result = Optional.empty(); + Optional metaData = getSnapshotMetaData(commitName); + if (metaData.isPresent()) { + String indexDirPath = metaData.get().getIndexDirPath(); + long gen = metaData.get().getGenerationNumber(); + + Directory d = solrCore.getDirectoryFactory().get(indexDirPath, DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_NONE); + try { + result = DirectoryReader.listCommits(d) + .stream() + .filter(ic -> ic.getGeneration() == gen) + .findAny(); + + if (!result.isPresent()) { + log.warn("Unable to find commit with generation {} in the directory {}", gen, indexDirPath); + } + + } finally { + solrCore.getDirectoryFactory().release(d); + } + } else { + log.warn("Commit with name {} is not persisted for core {}", commitName, solrCore.getName()); + } + + return result; + } + + private synchronized void persist() throws IOException { + String fileName = SNAPSHOTS_PREFIX + nextWriteGen; + IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT); + boolean success = false; + try { + CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); + out.writeVInt(nameToDetailsMapping.size()); + for(Entry ent : nameToDetailsMapping.entrySet()) { + out.writeString(ent.getKey()); + out.writeString(ent.getValue().getIndexDirPath()); + out.writeVLong(ent.getValue().getGenerationNumber()); + } + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(out); + IOUtils.deleteFilesIgnoringExceptions(dir, fileName); + } else { + IOUtils.close(out); + } + } + + dir.sync(Collections.singletonList(fileName)); + + if (nextWriteGen > 0) { + String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1); + // exception OK: likely it didn't exist + IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile); + } + + nextWriteGen++; + } + + private synchronized void deleteSnapshotMetadataFiles() throws IOException { + for(String file : dir.listAll()) { + if (file.startsWith(SNAPSHOTS_PREFIX)) { + dir.deleteFile(file); + } + } + } + + /** + * Reads the snapshot meta-data information from the given {@link Directory}. + */ + private synchronized void loadFromSnapshotMetadataFile() throws IOException { + log.info("Loading from snapshot metadata file..."); + long genLoaded = -1; + IOException ioe = null; + List snapshotFiles = new ArrayList<>(); + for(String file : dir.listAll()) { + if (file.startsWith(SNAPSHOTS_PREFIX)) { + long gen = Long.parseLong(file.substring(SNAPSHOTS_PREFIX.length())); + if (genLoaded == -1 || gen > genLoaded) { + snapshotFiles.add(file); + Map snapshotMetaDataMapping = new HashMap<>(); + IndexInput in = dir.openInput(file, IOContext.DEFAULT); + try { + CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); + int count = in.readVInt(); + for(int i=0;i 1) { + // Remove any broken / old snapshot files: + String curFileName = SNAPSHOTS_PREFIX + genLoaded; + for(String file : snapshotFiles) { + if (!curFileName.equals(file)) { + IOUtils.deleteFilesIgnoringExceptions(dir, file); + } + } + } + nextWriteGen = 1+genLoaded; + } + } +} diff --git a/solr/core/src/java/org/apache/solr/core/snapshots/package-info.java b/solr/core/src/java/org/apache/solr/core/snapshots/package-info.java new file mode 100644 index 00000000000..3242cd347f1 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/snapshots/package-info.java @@ -0,0 +1,22 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + +/** + * Core classes for Solr's persistent snapshots functionality + */ +package org.apache.solr.core.snapshots; \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java index 4b711873658..25b3b149389 100644 --- a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java @@ -160,7 +160,7 @@ public class BlobHandler extends RequestHandlerBase implements PluginInfoInitial } else { String q = "blobName:{0}"; if (version != -1) q = "id:{0}/{1}"; - QParser qparser = QParser.getParser(StrUtils.formatString(q, blobName, version), "lucene", req); + QParser qparser = QParser.getParser(StrUtils.formatString(q, blobName, version), req); final TopDocs docs = req.getSearcher().search(qparser.parse(), 1, new Sort(new SortField("version", SortField.Type.LONG, true))); if (docs.totalHits > 0) { rsp.add(ReplicationHandler.FILE_STREAM, new SolrCore.RawWriter() { diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrParams.java b/solr/core/src/java/org/apache/solr/handler/CdcrParams.java index aa1d5bfb553..3f65b90585f 100644 --- a/solr/core/src/java/org/apache/solr/handler/CdcrParams.java +++ b/solr/core/src/java/org/apache/solr/handler/CdcrParams.java @@ -121,6 +121,11 @@ public class CdcrParams { */ public final static String COUNTER_DELETES = "deletes"; + /** + * Counter for Bootstrap operations * + */ + public final static String COUNTER_BOOTSTRAP = "bootstraps"; + /** * A list of errors per target collection * */ @@ -165,7 +170,10 @@ public class CdcrParams { LASTPROCESSEDVERSION, QUEUES, OPS, - ERRORS; + ERRORS, + BOOTSTRAP, + BOOTSTRAP_STATUS, + CANCEL_BOOTSTRAP; public static CdcrAction get(String p) { if (p != null) { diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrReplicator.java b/solr/core/src/java/org/apache/solr/handler/CdcrReplicator.java index 66ce0966768..8519815f418 100644 --- a/solr/core/src/java/org/apache/solr/handler/CdcrReplicator.java +++ b/solr/core/src/java/org/apache/solr/handler/CdcrReplicator.java @@ -119,7 +119,7 @@ public class CdcrReplicator implements Runnable { // we might have read a single commit operation and reached the end of the update logs logReader.forwardSeek(subReader); - log.debug("Forwarded {} updates to target {}", counter, state.getTargetCollection()); + log.info("Forwarded {} updates to target {}", counter, state.getTargetCollection()); } catch (Exception e) { // report error and update error stats this.handleException(e); @@ -150,13 +150,13 @@ public class CdcrReplicator implements Runnable { if (e instanceof CdcrReplicatorException) { UpdateRequest req = ((CdcrReplicatorException) e).req; UpdateResponse rsp = ((CdcrReplicatorException) e).rsp; - log.warn("Failed to forward update request {}. Got response {}", req, rsp); + log.warn("Failed to forward update request {} to target: {}. Got response {}", req, state.getTargetCollection(), rsp); state.reportError(CdcrReplicatorState.ErrorType.BAD_REQUEST); } else if (e instanceof CloudSolrClient.RouteException) { - log.warn("Failed to forward update request", e); + log.warn("Failed to forward update request to target: " + state.getTargetCollection(), e); state.reportError(CdcrReplicatorState.ErrorType.BAD_REQUEST); } else { - log.warn("Failed to forward update request", e); + log.warn("Failed to forward update request to target: " + state.getTargetCollection(), e); state.reportError(CdcrReplicatorState.ErrorType.INTERNAL); } } diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java index af0161ed3c5..528e0b7aa6b 100644 --- a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java +++ b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java @@ -16,29 +16,49 @@ */ package org.apache.solr.handler; +import java.io.Closeable; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.http.client.HttpClient; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder; +import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SolrjNamedThreadFactory; import org.apache.solr.core.SolrCore; import org.apache.solr.update.CdcrUpdateLog; +import org.apache.solr.util.TimeOut; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE_STATUS; + class CdcrReplicatorManager implements CdcrStateManager.CdcrStateObserver { + private static final int MAX_BOOTSTRAP_ATTEMPTS = 5; + private static final int BOOTSTRAP_RETRY_DELAY_MS = 2000; + // 6 hours is hopefully long enough for most indexes + private static final long BOOTSTRAP_TIMEOUT_SECONDS = 6L * 3600L * 3600L; + private List replicatorStates; private final CdcrReplicatorScheduler scheduler; @@ -48,6 +68,9 @@ class CdcrReplicatorManager implements CdcrStateManager.CdcrStateObserver { private SolrCore core; private String path; + private ExecutorService bootstrapExecutor; + private volatile BootstrapStatusRunnable bootstrapStatusRunnable; + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); CdcrReplicatorManager(final SolrCore core, String path, @@ -104,12 +127,20 @@ class CdcrReplicatorManager implements CdcrStateManager.CdcrStateObserver { @Override public synchronized void stateUpdate() { if (leaderStateManager.amILeader() && processStateManager.getState().equals(CdcrParams.ProcessState.STARTED)) { + if (replicatorStates.size() > 0) { + this.bootstrapExecutor = ExecutorUtil.newMDCAwareFixedThreadPool(replicatorStates.size(), + new SolrjNamedThreadFactory("cdcr-bootstrap-status")); + } this.initLogReaders(); this.scheduler.start(); return; } this.scheduler.shutdown(); + if (bootstrapExecutor != null) { + IOUtils.closeQuietly(bootstrapStatusRunnable); + ExecutorUtil.shutdownAndAwaitTermination(bootstrapExecutor); + } this.closeLogReaders(); } @@ -117,7 +148,7 @@ class CdcrReplicatorManager implements CdcrStateManager.CdcrStateObserver { return replicatorStates; } - void initLogReaders() { + private void initLogReaders() { String collectionName = core.getCoreDescriptor().getCloudDescriptor().getCollectionName(); String shard = core.getCoreDescriptor().getCloudDescriptor().getShardId(); CdcrUpdateLog ulog = (CdcrUpdateLog) core.getUpdateHandler().getUpdateLog(); @@ -129,8 +160,23 @@ class CdcrReplicatorManager implements CdcrStateManager.CdcrStateObserver { log.info("Create new update log reader for target {} with checkpoint {} @ {}:{}", state.getTargetCollection(), checkpoint, collectionName, shard); CdcrUpdateLog.CdcrLogReader reader = ulog.newLogReader(); - reader.seek(checkpoint); + boolean seek = reader.seek(checkpoint); state.init(reader); + if (!seek) { + // targetVersion is lower than the oldest known entry. + // In this scenario, it probably means that there is a gap in the updates log. + // the best we can do here is to bootstrap the target leader by replicating the full index + final String targetCollection = state.getTargetCollection(); + state.setBootstrapInProgress(true); + log.info("Attempting to bootstrap target collection: {}, shard: {}", targetCollection, shard); + bootstrapStatusRunnable = new BootstrapStatusRunnable(core, state); + log.info("Submitting bootstrap task to executor"); + try { + bootstrapExecutor.submit(bootstrapStatusRunnable); + } catch (Exception e) { + log.error("Unable to submit bootstrap call to executor", e); + } + } } catch (IOException | SolrServerException | SolrException e) { log.warn("Unable to instantiate the log reader for target collection " + state.getTargetCollection(), e); } catch (InterruptedException e) { @@ -164,11 +210,203 @@ class CdcrReplicatorManager implements CdcrStateManager.CdcrStateObserver { */ void shutdown() { this.scheduler.shutdown(); + if (bootstrapExecutor != null) { + IOUtils.closeQuietly(bootstrapStatusRunnable); + ExecutorUtil.shutdownAndAwaitTermination(bootstrapExecutor); + } for (CdcrReplicatorState state : replicatorStates) { state.shutdown(); } replicatorStates.clear(); } + private class BootstrapStatusRunnable implements Runnable, Closeable { + private final CdcrReplicatorState state; + private final String targetCollection; + private final String shard; + private final String collectionName; + private final CdcrUpdateLog ulog; + private final String myCoreUrl; + + private volatile boolean closed = false; + + BootstrapStatusRunnable(SolrCore core, CdcrReplicatorState state) { + this.collectionName = core.getCoreDescriptor().getCloudDescriptor().getCollectionName(); + this.shard = core.getCoreDescriptor().getCloudDescriptor().getShardId(); + this.ulog = (CdcrUpdateLog) core.getUpdateHandler().getUpdateLog(); + this.state = state; + this.targetCollection = state.getTargetCollection(); + String baseUrl = core.getCoreDescriptor().getCoreContainer().getZkController().getBaseUrl(); + this.myCoreUrl = ZkCoreNodeProps.getCoreUrl(baseUrl, core.getName()); + } + + @Override + public void close() throws IOException { + closed = true; + try { + Replica leader = state.getClient().getZkStateReader().getLeaderRetry(targetCollection, shard, 30000); // assume same shard exists on target + String leaderCoreUrl = leader.getCoreUrl(); + HttpClient httpClient = state.getClient().getLbClient().getHttpClient(); + try (HttpSolrClient client = new HttpSolrClient.Builder(leaderCoreUrl).withHttpClient(httpClient).build()) { + sendCdcrCommand(client, CdcrParams.CdcrAction.CANCEL_BOOTSTRAP); + } catch (SolrServerException e) { + log.error("Error sending cancel bootstrap message to target collection: {} shard: {} leader: {}", + targetCollection, shard, leaderCoreUrl); + } + } catch (InterruptedException e) { + log.error("Interrupted while closing BootstrapStatusRunnable", e); + Thread.currentThread().interrupt(); + } + } + + @Override + public void run() { + int retries = 1; + boolean success = false; + try { + while (!closed && sendBootstrapCommand() != BootstrapStatus.SUBMITTED) { + Thread.sleep(BOOTSTRAP_RETRY_DELAY_MS); + } + TimeOut timeOut = new TimeOut(BOOTSTRAP_TIMEOUT_SECONDS, TimeUnit.SECONDS); + while (!timeOut.hasTimedOut()) { + if (closed) { + log.warn("Cancelling waiting for bootstrap on target: {} shard: {} to complete", targetCollection, shard); + state.setBootstrapInProgress(false); + break; + } + BootstrapStatus status = getBoostrapStatus(); + if (status == BootstrapStatus.RUNNING) { + try { + log.info("CDCR bootstrap running for {} seconds, sleeping for {} ms", + BOOTSTRAP_TIMEOUT_SECONDS - timeOut.timeLeft(TimeUnit.SECONDS), BOOTSTRAP_RETRY_DELAY_MS); + Thread.sleep(BOOTSTRAP_RETRY_DELAY_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } else if (status == BootstrapStatus.COMPLETED) { + log.info("CDCR bootstrap successful in {} seconds", BOOTSTRAP_TIMEOUT_SECONDS - timeOut.timeLeft(TimeUnit.SECONDS)); + long checkpoint = CdcrReplicatorManager.this.getCheckpoint(state); + log.info("Create new update log reader for target {} with checkpoint {} @ {}:{}", state.getTargetCollection(), + checkpoint, collectionName, shard); + CdcrUpdateLog.CdcrLogReader reader1 = ulog.newLogReader(); + reader1.seek(checkpoint); + success = true; + break; + } else if (status == BootstrapStatus.FAILED) { + log.warn("CDCR bootstrap failed in {} seconds", BOOTSTRAP_TIMEOUT_SECONDS - timeOut.timeLeft(TimeUnit.SECONDS)); + // let's retry a fixed number of times before giving up + if (retries >= MAX_BOOTSTRAP_ATTEMPTS) { + log.error("Unable to bootstrap the target collection: {}, shard: {} even after {} retries", targetCollection, shard, retries); + break; + } else { + log.info("Retry: {} - Attempting to bootstrap target collection: {} shard: {}", retries, targetCollection, shard); + while (!closed && sendBootstrapCommand() != BootstrapStatus.SUBMITTED) { + Thread.sleep(BOOTSTRAP_RETRY_DELAY_MS); + } + timeOut = new TimeOut(BOOTSTRAP_TIMEOUT_SECONDS, TimeUnit.SECONDS); // reset the timer + retries++; + } + } else if (status == BootstrapStatus.NOTFOUND) { + // the leader of the target shard may have changed and therefore there is no record of the + // bootstrap process so we must retry the operation + while (!closed && sendBootstrapCommand() != BootstrapStatus.SUBMITTED) { + Thread.sleep(BOOTSTRAP_RETRY_DELAY_MS); + } + retries = 1; + timeOut = new TimeOut(6L * 3600L * 3600L, TimeUnit.SECONDS); // reset the timer + } else if (status == BootstrapStatus.UNKNOWN) { + // we were not able to query the status on the remote end + // so just sleep for a bit and try again + Thread.sleep(BOOTSTRAP_RETRY_DELAY_MS); + } + } + } catch (InterruptedException e) { + log.info("Bootstrap thread interrupted"); + state.reportError(CdcrReplicatorState.ErrorType.INTERNAL); + Thread.currentThread().interrupt(); + } catch (IOException | SolrServerException | SolrException e) { + log.error("Unable to bootstrap the target collection " + targetCollection + " shard: " + shard, e); + state.reportError(CdcrReplicatorState.ErrorType.BAD_REQUEST); + } finally { + if (success) { + log.info("Bootstrap successful, giving the go-ahead to replicator"); + state.setBootstrapInProgress(false); + } + } + } + + private BootstrapStatus sendBootstrapCommand() throws InterruptedException { + Replica leader = state.getClient().getZkStateReader().getLeaderRetry(targetCollection, shard, 30000); // assume same shard exists on target + String leaderCoreUrl = leader.getCoreUrl(); + HttpClient httpClient = state.getClient().getLbClient().getHttpClient(); + try (HttpSolrClient client = new HttpSolrClient.Builder(leaderCoreUrl).withHttpClient(httpClient).build()) { + log.info("Attempting to bootstrap target collection: {} shard: {} leader: {}", targetCollection, shard, leaderCoreUrl); + try { + NamedList response = sendCdcrCommand(client, CdcrParams.CdcrAction.BOOTSTRAP, ReplicationHandler.MASTER_URL, myCoreUrl); + log.debug("CDCR Bootstrap response: {}", response); + String status = response.get(RESPONSE_STATUS).toString(); + return BootstrapStatus.valueOf(status.toUpperCase(Locale.ROOT)); + } catch (Exception e) { + log.error("Exception submitting bootstrap request", e); + return BootstrapStatus.UNKNOWN; + } + } catch (IOException e) { + log.error("There shouldn't be an IOException while closing but there was!", e); + } + return BootstrapStatus.UNKNOWN; + } + + private BootstrapStatus getBoostrapStatus() throws InterruptedException { + try { + Replica leader = state.getClient().getZkStateReader().getLeaderRetry(targetCollection, shard, 30000); // assume same shard exists on target + String leaderCoreUrl = leader.getCoreUrl(); + HttpClient httpClient = state.getClient().getLbClient().getHttpClient(); + try (HttpSolrClient client = new HttpSolrClient.Builder(leaderCoreUrl).withHttpClient(httpClient).build()) { + NamedList response = sendCdcrCommand(client, CdcrParams.CdcrAction.BOOTSTRAP_STATUS); + String status = (String) response.get(RESPONSE_STATUS); + BootstrapStatus bootstrapStatus = BootstrapStatus.valueOf(status.toUpperCase(Locale.ROOT)); + if (bootstrapStatus == BootstrapStatus.RUNNING) { + return BootstrapStatus.RUNNING; + } else if (bootstrapStatus == BootstrapStatus.COMPLETED) { + return BootstrapStatus.COMPLETED; + } else if (bootstrapStatus == BootstrapStatus.FAILED) { + return BootstrapStatus.FAILED; + } else if (bootstrapStatus == BootstrapStatus.NOTFOUND) { + log.warn("Bootstrap process was not found on target collection: {} shard: {}, leader: {}", targetCollection, shard, leaderCoreUrl); + return BootstrapStatus.NOTFOUND; + } else if (bootstrapStatus == BootstrapStatus.CANCELLED) { + return BootstrapStatus.CANCELLED; + } else { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Unknown status: " + status + " returned by BOOTSTRAP_STATUS command"); + } + } + } catch (Exception e) { + log.error("Exception during bootstrap status request", e); + return BootstrapStatus.UNKNOWN; + } + } + } + + private NamedList sendCdcrCommand(SolrClient client, CdcrParams.CdcrAction action, String... params) throws SolrServerException, IOException { + ModifiableSolrParams solrParams = new ModifiableSolrParams(); + solrParams.set(CommonParams.QT, "/cdcr"); + solrParams.set(CommonParams.ACTION, action.toString()); + for (int i = 0; i < params.length - 1; i+=2) { + solrParams.set(params[i], params[i + 1]); + } + SolrRequest request = new QueryRequest(solrParams); + return client.request(request); + } + + private enum BootstrapStatus { + SUBMITTED, + RUNNING, + COMPLETED, + FAILED, + NOTFOUND, + CANCELLED, + UNKNOWN + } } diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorScheduler.java b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorScheduler.java index bb817e5df2a..62abeab8362 100644 --- a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorScheduler.java +++ b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorScheduler.java @@ -77,7 +77,11 @@ class CdcrReplicatorScheduler { CdcrReplicatorState state = statesQueue.poll(); assert state != null; // Should never happen try { - new CdcrReplicator(state, batchSize).run(); + if (!state.isBootstrapInProgress()) { + new CdcrReplicator(state, batchSize).run(); + } else { + log.debug("Replicator state is bootstrapping, skipping replication for target collection {}", state.getTargetCollection()); + } } finally { statesQueue.offer(state); } diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorState.java b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorState.java index 9e01f117b67..2ca0d800be5 100644 --- a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorState.java +++ b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorState.java @@ -27,6 +27,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.update.CdcrUpdateLog; @@ -53,6 +55,9 @@ class CdcrReplicatorState { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final AtomicBoolean bootstrapInProgress = new AtomicBoolean(false); + private final AtomicInteger numBootstraps = new AtomicInteger(); + CdcrReplicatorState(final String targetCollection, final String zkHost, final CloudSolrClient targetClient) { this.targetCollection = targetCollection; this.targetClient = targetClient; @@ -164,6 +169,24 @@ class CdcrReplicatorState { return this.benchmarkTimer; } + /** + * @return true if a bootstrap operation is in progress, false otherwise + */ + boolean isBootstrapInProgress() { + return bootstrapInProgress.get(); + } + + void setBootstrapInProgress(boolean inProgress) { + if (bootstrapInProgress.compareAndSet(true, false)) { + numBootstraps.incrementAndGet(); + } + bootstrapInProgress.set(inProgress); + } + + public int getNumBootstraps() { + return numBootstraps.get(); + } + enum ErrorType { INTERNAL, BAD_REQUEST; diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/CdcrRequestHandler.java index f60e5621bfb..f706637bb53 100644 --- a/solr/core/src/java/org/apache/solr/handler/CdcrRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/CdcrRequestHandler.java @@ -16,6 +16,7 @@ */ package org.apache.solr.handler; +import java.io.Closeable; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -24,14 +25,20 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; @@ -41,21 +48,33 @@ import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.params.UpdateParams; import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.CloseHook; import org.apache.solr.core.PluginBag; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.update.CdcrUpdateLog; import org.apache.solr.update.UpdateLog; +import org.apache.solr.update.VersionInfo; +import org.apache.solr.update.processor.DistributedUpdateProcessor; import org.apache.solr.util.DefaultSolrThreadFactory; import org.apache.solr.util.plugin.SolrCoreAware; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.solr.handler.admin.CoreAdminHandler.COMPLETED; +import static org.apache.solr.handler.admin.CoreAdminHandler.FAILED; +import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE; +import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE_MESSAGE; +import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE_STATUS; +import static org.apache.solr.handler.admin.CoreAdminHandler.RUNNING; + /** *

* This request handler implements the CDCR API and is responsible of the execution of the @@ -199,6 +218,18 @@ public class CdcrRequestHandler extends RequestHandlerBase implements SolrCoreAw this.handleErrorsAction(req, rsp); break; } + case BOOTSTRAP: { + this.handleBootstrapAction(req, rsp); + break; + } + case BOOTSTRAP_STATUS: { + this.handleBootstrapStatus(req, rsp); + break; + } + case CANCEL_BOOTSTRAP: { + this.handleCancelBootstrap(req, rsp); + break; + } default: { throw new RuntimeException("Unknown action: " + action); } @@ -409,10 +440,20 @@ public class CdcrRequestHandler extends RequestHandlerBase implements SolrCoreAw } UpdateLog ulog = core.getUpdateHandler().getUpdateLog(); + VersionInfo versionInfo = ulog.getVersionInfo(); try (UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates()) { - List versions = recentUpdates.getVersions(1); - long lastVersion = versions.isEmpty() ? -1 : Math.abs(versions.get(0)); - rsp.add(CdcrParams.CHECKPOINT, lastVersion); + long maxVersionFromRecent = recentUpdates.getMaxRecentVersion(); + long maxVersionFromIndex = versionInfo.getMaxVersionFromIndex(req.getSearcher()); + log.info("Found maxVersionFromRecent {} maxVersionFromIndex {}", maxVersionFromRecent, maxVersionFromIndex); + // there is no race with ongoing bootstrap because we don't expect any updates to come from the source + long maxVersion = Math.max(maxVersionFromIndex, maxVersionFromRecent); + if (maxVersion == 0L) { + maxVersion = -1; + } + rsp.add(CdcrParams.CHECKPOINT, maxVersion); + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Action '" + CdcrParams.CdcrAction.SHARDCHECKPOINT + + "' could not read max version"); } } @@ -574,6 +615,192 @@ public class CdcrRequestHandler extends RequestHandlerBase implements SolrCoreAw rsp.add(CdcrParams.ERRORS, hosts); } + private AtomicBoolean running = new AtomicBoolean(); + private volatile Future bootstrapFuture; + private volatile BootstrapCallable bootstrapCallable; + + private void handleBootstrapAction(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, SolrServerException { + String collectionName = core.getCoreDescriptor().getCloudDescriptor().getCollectionName(); + String shard = core.getCoreDescriptor().getCloudDescriptor().getShardId(); + if (!leaderStateManager.amILeader()) { + log.warn("Action {} sent to non-leader replica @ {}:{}", CdcrParams.CdcrAction.BOOTSTRAP, collectionName, shard); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Action " + CdcrParams.CdcrAction.BOOTSTRAP + + " sent to non-leader replica"); + } + + Runnable runnable = () -> { + Lock recoveryLock = req.getCore().getSolrCoreState().getRecoveryLock(); + boolean locked = recoveryLock.tryLock(); + try { + if (!locked) { + handleCancelBootstrap(req, rsp); + } else if (leaderStateManager.amILeader()) { + running.set(true); + String masterUrl = req.getParams().get(ReplicationHandler.MASTER_URL); + bootstrapCallable = new BootstrapCallable(masterUrl, core); + bootstrapFuture = core.getCoreDescriptor().getCoreContainer().getUpdateShardHandler().getRecoveryExecutor().submit(bootstrapCallable); + try { + bootstrapFuture.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.warn("Bootstrap was interrupted", e); + } catch (ExecutionException e) { + log.error("Bootstrap operation failed", e); + } + } else { + log.error("Action {} sent to non-leader replica @ {}:{}. Aborting bootstrap.", CdcrParams.CdcrAction.BOOTSTRAP, collectionName, shard); + } + } finally { + if (locked) { + running.set(false); + recoveryLock.unlock(); + } + } + }; + + try { + core.getCoreDescriptor().getCoreContainer().getUpdateShardHandler().getUpdateExecutor().submit(runnable); + rsp.add(RESPONSE_STATUS, "submitted"); + } catch (RejectedExecutionException ree) { + // no problem, we're probably shutting down + rsp.add(RESPONSE_STATUS, "failed"); + } + } + + private void handleCancelBootstrap(SolrQueryRequest req, SolrQueryResponse rsp) { + BootstrapCallable callable = this.bootstrapCallable; + IOUtils.closeQuietly(callable); + rsp.add(RESPONSE_STATUS, "cancelled"); + } + + private void handleBootstrapStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, SolrServerException { + if (running.get()) { + rsp.add(RESPONSE_STATUS, RUNNING); + return; + } + + Future future = bootstrapFuture; + BootstrapCallable callable = this.bootstrapCallable; + if (future == null) { + rsp.add(RESPONSE_STATUS, "notfound"); + rsp.add(RESPONSE_MESSAGE, "No bootstrap found in running, completed or failed states"); + } else if (future.isCancelled() || callable.isClosed()) { + rsp.add(RESPONSE_STATUS, "cancelled"); + } else if (future.isDone()) { + // could be a normal termination or an exception + try { + Boolean result = future.get(); + if (result) { + rsp.add(RESPONSE_STATUS, COMPLETED); + } else { + rsp.add(RESPONSE_STATUS, FAILED); + } + } catch (InterruptedException e) { + // should not happen? + } catch (ExecutionException e) { + rsp.add(RESPONSE_STATUS, FAILED); + rsp.add(RESPONSE, e); + } catch (CancellationException ce) { + rsp.add(RESPONSE_STATUS, FAILED); + rsp.add(RESPONSE_MESSAGE, "Bootstrap was cancelled"); + } + } else { + rsp.add(RESPONSE_STATUS, RUNNING); + } + } + + private static class BootstrapCallable implements Callable, Closeable { + private final String masterUrl; + private final SolrCore core; + private volatile boolean closed = false; + + BootstrapCallable(String masterUrl, SolrCore core) { + this.masterUrl = masterUrl; + this.core = core; + } + + @Override + public void close() throws IOException { + closed = true; + SolrRequestHandler handler = core.getRequestHandler(ReplicationHandler.PATH); + ReplicationHandler replicationHandler = (ReplicationHandler) handler; + replicationHandler.abortFetch(); + } + + public boolean isClosed() { + return closed; + } + + @Override + public Boolean call() throws Exception { + boolean success = false; + UpdateLog ulog = core.getUpdateHandler().getUpdateLog(); + // we start buffering updates as a safeguard however we do not expect + // to receive any updates from the source during bootstrap + ulog.bufferUpdates(); + try { + commitOnLeader(masterUrl); + // use rep handler directly, so we can do this sync rather than async + SolrRequestHandler handler = core.getRequestHandler(ReplicationHandler.PATH); + ReplicationHandler replicationHandler = (ReplicationHandler) handler; + + if (replicationHandler == null) { + throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, + "Skipping recovery, no " + ReplicationHandler.PATH + " handler found"); + } + + ModifiableSolrParams solrParams = new ModifiableSolrParams(); + solrParams.set(ReplicationHandler.MASTER_URL, masterUrl); + // we do not want the raw tlog files from the source + solrParams.set(ReplicationHandler.TLOG_FILES, false); + + success = replicationHandler.doFetch(solrParams, false); + + // this is required because this callable can race with HttpSolrCall#destroy + // which clears the request info. + // Applying buffered updates fails without the following line because LogReplayer + // also tries to set request info and fails with AssertionError + SolrRequestInfo.clearRequestInfo(); + + Future future = ulog.applyBufferedUpdates(); + if (future == null) { + // no replay needed + log.info("No replay needed."); + } else { + log.info("Replaying buffered documents."); + // wait for replay + UpdateLog.RecoveryInfo report = future.get(); + if (report.failed) { + SolrException.log(log, "Replay failed"); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Replay failed"); + } + } + return success; + } finally { + if (closed || !success) { + // we cannot apply the buffer in this case because it will introduce newer versions in the + // update log and then the source cluster will get those versions via collectioncheckpoint + // causing the versions in between to be completely missed + boolean dropped = ulog.dropBufferedUpdates(); + assert dropped; + } + } + } + + private void commitOnLeader(String leaderUrl) throws SolrServerException, + IOException { + try (HttpSolrClient client = new HttpSolrClient.Builder(leaderUrl).build()) { + client.setConnectionTimeout(30000); + UpdateRequest ureq = new UpdateRequest(); + ureq.setParams(new ModifiableSolrParams()); + ureq.getParams().set(DistributedUpdateProcessor.COMMIT_END_POINT, true); + ureq.getParams().set(UpdateParams.OPEN_SEARCHER, false); + ureq.setAction(AbstractUpdateRequest.ACTION.COMMIT, false, true).process( + client); + } + } + } + @Override public String getDescription() { return "Manage Cross Data Center Replication"; diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index 714b8004d5e..080cf9f360c 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -82,6 +82,9 @@ import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.IndexDeletionPolicyWrapper; import org.apache.solr.core.SolrCore; +import org.apache.solr.core.snapshots.SolrSnapshotManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData; import org.apache.solr.handler.ReplicationHandler.*; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; @@ -468,9 +471,18 @@ public class IndexFetcher { // let the system know we are changing dir's and the old one // may be closed if (indexDir != null) { - LOG.info("removing old index directory " + indexDir); solrCore.getDirectoryFactory().doneWithDirectory(indexDir); - solrCore.getDirectoryFactory().remove(indexDir); + + SolrSnapshotMetaDataManager snapshotsMgr = solrCore.getSnapshotMetaDataManager(); + Collection snapshots = snapshotsMgr.listSnapshotsInIndexDir(indexDirPath); + + // Delete the old index directory only if no snapshot exists in that directory. + if(snapshots.isEmpty()) { + LOG.info("removing old index directory " + indexDir); + solrCore.getDirectoryFactory().remove(indexDir); + } else { + SolrSnapshotManager.deleteNonSnapshotIndexFiles(indexDir, snapshots); + } } } @@ -738,14 +750,14 @@ public class IndexFetcher { } private void openNewSearcherAndUpdateCommitPoint() throws IOException { - SolrQueryRequest req = new LocalSolrQueryRequest(solrCore, - new ModifiableSolrParams()); - RefCounted searcher = null; IndexCommit commitPoint; + // must get the latest solrCore object because the one we have might be closed because of a reload + // todo stop keeping solrCore around + SolrCore core = solrCore.getCoreDescriptor().getCoreContainer().getCore(solrCore.getName()); try { Future[] waitSearcher = new Future[1]; - searcher = solrCore.getSearcher(true, true, waitSearcher, true); + searcher = core.getSearcher(true, true, waitSearcher, true); if (waitSearcher[0] != null) { try { waitSearcher[0].get(); @@ -755,10 +767,10 @@ public class IndexFetcher { } commitPoint = searcher.get().getIndexReader().getIndexCommit(); } finally { - req.close(); if (searcher != null) { searcher.decref(); } + core.close(); } // update the commit point in replication handler diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index 3f231b2fca3..dcd70f4d79a 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -125,7 +125,7 @@ public class MoreLikeThisHandler extends RequestHandlerBase filters = new ArrayList<>(); for (String fq : fqs) { if (fq != null && fq.trim().length() != 0) { - QParser fqp = QParser.getParser(fq, null, req); + QParser fqp = QParser.getParser(fq, req); filters.add(fqp.getQuery()); } } diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index ff93c42d18b..aee3b972d58 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -87,6 +87,7 @@ import org.apache.solr.core.SolrDeletionPolicy; import org.apache.solr.core.SolrEventListener; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.core.backup.repository.LocalFileSystemRepository; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.search.SolrIndexSearcher; @@ -299,9 +300,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw rsp.add("message","No slave configured"); } } else if (command.equalsIgnoreCase(CMD_ABORT_FETCH)) { - IndexFetcher fetcher = currentIndexFetcher; - if (fetcher != null){ - fetcher.abortFetch(); + if (abortFetch()){ rsp.add(STATUS, OK_STATUS); } else { rsp.add(STATUS,ERR_STATUS); @@ -320,6 +319,16 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw } } + public boolean abortFetch() { + IndexFetcher fetcher = currentIndexFetcher; + if (fetcher != null){ + fetcher.abortFetch(); + return true; + } else { + return false; + } + } + private void deleteSnapshot(ModifiableSolrParams params) { String name = params.get(NAME); if(name == null) { @@ -512,11 +521,24 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw numberToKeep = Integer.MAX_VALUE; } - IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy(); - IndexCommit indexCommit = delPolicy.getLatestCommit(); + IndexCommit indexCommit = null; + String commitName = params.get(CoreAdminParams.COMMIT_NAME); + if (commitName != null) { + SolrSnapshotMetaDataManager snapshotMgr = core.getSnapshotMetaDataManager(); + Optional commit = snapshotMgr.getIndexCommitByName(commitName); + if(commit.isPresent()) { + indexCommit = commit.get(); + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to find an index commit with name " + commitName + + " for core " + core.getName()); + } + } else { + IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy(); + indexCommit = delPolicy.getLatestCommit(); - if (indexCommit == null) { - indexCommit = req.getSearcher().getIndexReader().getIndexCommit(); + if (indexCommit == null) { + indexCommit = req.getSearcher().getIndexReader().getIndexCommit(); + } } String location = params.get(CoreAdminParams.BACKUP_LOCATION); @@ -539,7 +561,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw } // small race here before the commit point is saved - SnapShooter snapShooter = new SnapShooter(repo, core, location, params.get(NAME)); + SnapShooter snapShooter = new SnapShooter(repo, core, location, params.get(NAME), commitName); snapShooter.validateCreateSnapshot(); snapShooter.createSnapAsync(indexCommit, numberToKeep, (nl) -> snapShootDetails = nl); @@ -644,7 +666,8 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw rsp.add(CMD_GET_FILE_LIST, result); // fetch list of tlog files only if cdcr is activated - if (core.getUpdateHandler().getUpdateLog() != null && core.getUpdateHandler().getUpdateLog() instanceof CdcrUpdateLog) { + if (solrParams.getBool(TLOG_FILES, true) && core.getUpdateHandler().getUpdateLog() != null + && core.getUpdateHandler().getUpdateLog() instanceof CdcrUpdateLog) { try { List> tlogfiles = getTlogFileList(commit); LOG.info("Adding tlog files to list: " + tlogfiles); diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java index bb7df32dc42..d28957c9617 100644 --- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java +++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java @@ -16,13 +16,17 @@ */ package org.apache.solr.handler; +import java.lang.invoke.MethodHandles; +import java.net.URL; +import java.util.concurrent.atomic.LongAdder; + import org.apache.solr.common.SolrException; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SuppressForbidden; -import org.apache.solr.core.PluginInfo; import org.apache.solr.core.PluginBag; +import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrInfoMBean; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; @@ -35,10 +39,6 @@ import org.apache.solr.util.stats.TimerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; -import java.net.URL; -import java.util.concurrent.atomic.AtomicLong; - import static org.apache.solr.core.RequestParams.USEPARAM; /** @@ -53,10 +53,10 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo protected boolean httpCaching = true; // Statistics - private final AtomicLong numRequests = new AtomicLong(); - private final AtomicLong numServerErrors = new AtomicLong(); - private final AtomicLong numClientErrors = new AtomicLong(); - private final AtomicLong numTimeouts = new AtomicLong(); + private final LongAdder numRequests = new LongAdder(); + private final LongAdder numServerErrors = new LongAdder(); + private final LongAdder numClientErrors = new LongAdder(); + private final LongAdder numTimeouts = new LongAdder(); private final Timer requestTimes = new Timer(); private final long handlerStart; @@ -144,7 +144,7 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo @Override public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) { - numRequests.incrementAndGet(); + numRequests.increment(); TimerContext timer = requestTimes.time(); try { if(pluginInfo != null && pluginInfo.attributes.containsKey(USEPARAM)) req.getContext().put(USEPARAM,pluginInfo.attributes.get(USEPARAM)); @@ -158,7 +158,7 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo Object partialResults = header.get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY); boolean timedOut = partialResults == null ? false : (Boolean)partialResults; if( timedOut ) { - numTimeouts.incrementAndGet(); + numTimeouts.increment(); rsp.setHttpCaching(false); } } @@ -185,9 +185,9 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo SolrException.log(log, e); if (isServerError) { - numServerErrors.incrementAndGet(); + numServerErrors.increment(); } else { - numClientErrors.incrementAndGet(); + numClientErrors.increment(); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java index d3c98fac432..6aef35c448a 100644 --- a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java +++ b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java @@ -19,6 +19,7 @@ package org.apache.solr.handler; import java.lang.invoke.MethodHandles; import java.net.URI; import java.text.SimpleDateFormat; +import java.util.Collection; import java.util.Date; import java.util.Locale; import java.util.concurrent.Callable; @@ -32,6 +33,9 @@ import org.apache.solr.common.SolrException; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.SolrCore; import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.core.snapshots.SolrSnapshotManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,6 +67,7 @@ public class RestoreCore implements Callable { String restoreIndexName = "restore." + dateFormat.format(new Date()); String restoreIndexPath = core.getDataDir() + restoreIndexName; + String indexDirPath = core.getIndexDir(); Directory restoreIndexDir = null; Directory indexDir = null; try { @@ -71,7 +76,7 @@ public class RestoreCore implements Callable { DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); //Prefer local copy. - indexDir = core.getDirectoryFactory().get(core.getIndexDir(), + indexDir = core.getDirectoryFactory().get(indexDirPath, DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); //Move all files from backupDir to restoreIndexDir @@ -130,7 +135,16 @@ public class RestoreCore implements Callable { } if (success) { core.getDirectoryFactory().doneWithDirectory(indexDir); - core.getDirectoryFactory().remove(indexDir); + + SolrSnapshotMetaDataManager snapshotsMgr = core.getSnapshotMetaDataManager(); + Collection snapshots = snapshotsMgr.listSnapshotsInIndexDir(indexDirPath); + + // Delete the old index directory only if no snapshot exists in that directory. + if (snapshots.isEmpty()) { + core.getDirectoryFactory().remove(indexDir); + } else { + SolrSnapshotManager.deleteNonSnapshotIndexFiles(indexDir, snapshots); + } } return true; diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java index 063c3d4d83c..35e463b44ca 100644 --- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java @@ -185,7 +185,7 @@ public class SchemaHandler extends RequestHandlerBase implements SolrCoreAware, if (parts.get(0).isEmpty()) parts.remove(0); if (parts.size() > 1 && level2.containsKey(parts.get(1))) { String realName = parts.get(1); - String fieldName = IndexSchema.SchemaProps.nameMapping.get(realName); + String fieldName = IndexSchema.nameMapping.get(realName); String pathParam = level2.get(realName); if (parts.size() > 2) { diff --git a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java index 5ac324329a0..e12649dcd91 100644 --- a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java +++ b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java @@ -26,12 +26,14 @@ import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Locale; +import java.util.Optional; import java.util.function.Consumer; import com.google.common.base.Preconditions; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.store.Directory; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.IndexDeletionPolicyWrapper; @@ -39,6 +41,7 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.core.backup.repository.BackupRepository.PathType; import org.apache.solr.core.backup.repository.LocalFileSystemRepository; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.RefCounted; import org.slf4j.Logger; @@ -59,6 +62,7 @@ public class SnapShooter { private URI baseSnapDirPath = null; private URI snapshotDirPath = null; private BackupRepository backupRepo = null; + private String commitName; // can be null @Deprecated public SnapShooter(SolrCore core, String location, String snapshotName) { @@ -71,14 +75,14 @@ public class SnapShooter { } else { snapDirStr = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString(); } - initialize(new LocalFileSystemRepository(), core, snapDirStr, snapshotName); + initialize(new LocalFileSystemRepository(), core, snapDirStr, snapshotName, null); } - public SnapShooter(BackupRepository backupRepo, SolrCore core, String location, String snapshotName) { - initialize(backupRepo, core, location, snapshotName); + public SnapShooter(BackupRepository backupRepo, SolrCore core, String location, String snapshotName, String commitName) { + initialize(backupRepo, core, location, snapshotName, commitName); } - private void initialize(BackupRepository backupRepo, SolrCore core, String location, String snapshotName) { + private void initialize(BackupRepository backupRepo, SolrCore core, String location, String snapshotName, String commitName) { this.solrCore = Preconditions.checkNotNull(core); this.backupRepo = Preconditions.checkNotNull(backupRepo); this.baseSnapDirPath = backupRepo.createURI(Preconditions.checkNotNull(location)).normalize(); @@ -90,6 +94,7 @@ public class SnapShooter { directoryName = "snapshot." + fmt.format(new Date()); } this.snapshotDirPath = backupRepo.createURI(location, directoryName); + this.commitName = commitName; } public BackupRepository getBackupRepository() { @@ -145,16 +150,26 @@ public class SnapShooter { } public NamedList createSnapshot() throws Exception { - IndexDeletionPolicyWrapper deletionPolicy = solrCore.getDeletionPolicy(); RefCounted searcher = solrCore.getSearcher(); try { - //TODO should we try solrCore.getDeletionPolicy().getLatestCommit() first? - IndexCommit indexCommit = searcher.get().getIndexReader().getIndexCommit(); - deletionPolicy.saveCommitPoint(indexCommit.getGeneration()); - try { - return createSnapshot(indexCommit); - } finally { - deletionPolicy.releaseCommitPoint(indexCommit.getGeneration()); + if (commitName != null) { + SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager(); + Optional commit = snapshotMgr.getIndexCommitByName(commitName); + if(commit.isPresent()) { + return createSnapshot(commit.get()); + } + throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to find an index commit with name " + commitName + + " for core " + solrCore.getName()); + } else { + //TODO should we try solrCore.getDeletionPolicy().getLatestCommit() first? + IndexDeletionPolicyWrapper deletionPolicy = solrCore.getDeletionPolicy(); + IndexCommit indexCommit = searcher.get().getIndexReader().getIndexCommit(); + deletionPolicy.saveCommitPoint(indexCommit.getGeneration()); + try { + return createSnapshot(indexCommit); + } finally { + deletionPolicy.releaseCommitPoint(indexCommit.getGeneration()); + } } } finally { searcher.decref(); diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java index b34cff59d42..e97df34eb9d 100644 --- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java @@ -122,6 +122,8 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware, .withFunctionName("intersect", IntersectStream.class) .withFunctionName("complement", ComplementStream.class) .withFunctionName("sort", SortStream.class) + .withFunctionName("train", TextLogitStream.class) + .withFunctionName("features", FeaturesSelectionStream.class) .withFunctionName("daemon", DaemonStream.class) .withFunctionName("shortestPath", ShortestPathStream.class) .withFunctionName("gatherNodes", GatherNodesStream.class) diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 2e906aed55b..a9703f30a1c 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -714,10 +714,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission String location = repository.getBackupLocation(req.getParams().get(CoreAdminParams.BACKUP_LOCATION)); if (location == null) { //Refresh the cluster property file to make sure the value set for location is the latest - h.coreContainer.getZkController().getZkStateReader().forceUpdateClusterProperties(); - // Check if the location is specified in the cluster property. - location = h.coreContainer.getZkController().getZkStateReader().getClusterProperty(CoreAdminParams.BACKUP_LOCATION, null); + location = new ClusterProperties(h.coreContainer.getZkController().getZkClient()).getClusterProperty(CoreAdminParams.BACKUP_LOCATION, null); if (location == null) { throw new SolrException(ErrorCode.BAD_REQUEST, "'location' is not specified as a query" + " parameter or as a default repository property or as a cluster property."); @@ -755,10 +753,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission String location = repository.getBackupLocation(req.getParams().get(CoreAdminParams.BACKUP_LOCATION)); if (location == null) { //Refresh the cluster property file to make sure the value set for location is the latest - h.coreContainer.getZkController().getZkStateReader().forceUpdateClusterProperties(); - // Check if the location is specified in the cluster property. - location = h.coreContainer.getZkController().getZkStateReader().getClusterProperty("location", null); + location = new ClusterProperties(h.coreContainer.getZkController().getZkClient()).getClusterProperty("location", null); if (location == null) { throw new SolrException(ErrorCode.BAD_REQUEST, "'location' is not specified as a query" + " parameter or as a default repository property or as a cluster property."); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java index 9b9aafa6ae3..e4103c54e17 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java @@ -34,6 +34,7 @@ import java.util.concurrent.Future; import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; @@ -59,9 +60,13 @@ import org.apache.solr.core.CachingDirectoryFactory; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.DirectoryFactory; +import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.core.backup.repository.BackupRepository; +import org.apache.solr.core.snapshots.SolrSnapshotManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData; import org.apache.solr.handler.RestoreCore; import org.apache.solr.handler.SnapShooter; import org.apache.solr.handler.admin.CoreAdminHandler.CoreAdminOp; @@ -794,22 +799,26 @@ enum CoreAdminOperation implements CoreAdminOp { + " parameter or as a default repository property"); } - try (SolrCore core = it.handler.coreContainer.getCore(cname)) { - SnapShooter snapShooter = new SnapShooter(repository, core, location, name); - // validateCreateSnapshot will create parent dirs instead of throw; that choice is dubious. - // But we want to throw. One reason is that - // this dir really should, in fact must, already exist here if triggered via a collection backup on a shared - // file system. Otherwise, perhaps the FS location isn't shared -- we want an error. - if (!snapShooter.getBackupRepository().exists(snapShooter.getLocation())) { - throw new SolrException(ErrorCode.BAD_REQUEST, - "Directory to contain snapshots doesn't exist: " + snapShooter.getLocation()); + // An optional parameter to describe the snapshot to be backed-up. If this + // parameter is not supplied, the latest index commit is backed-up. + String commitName = params.get(CoreAdminParams.COMMIT_NAME); + + try (SolrCore core = it.handler.coreContainer.getCore(cname)) { + SnapShooter snapShooter = new SnapShooter(repository, core, location, name, commitName); + // validateCreateSnapshot will create parent dirs instead of throw; that choice is dubious. + // But we want to throw. One reason is that + // this dir really should, in fact must, already exist here if triggered via a collection backup on a shared + // file system. Otherwise, perhaps the FS location isn't shared -- we want an error. + if (!snapShooter.getBackupRepository().exists(snapShooter.getLocation())) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Directory to contain snapshots doesn't exist: " + snapShooter.getLocation()); + } + snapShooter.validateCreateSnapshot(); + snapShooter.createSnapshot(); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed to backup core=" + cname + " because " + e, e); } - snapShooter.validateCreateSnapshot(); - snapShooter.createSnapshot(); - } catch (Exception e) { - throw new SolrException(ErrorCode.SERVER_ERROR, - "Failed to backup core=" + cname + " because " + e, e); - } }), RESTORECORE_OP(RESTORECORE, it -> { @@ -845,6 +854,92 @@ enum CoreAdminOperation implements CoreAdminOp { throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to restore core=" + core.getName()); } } + }), + CREATESNAPSHOT_OP(CREATESNAPSHOT, it -> { + CoreContainer cc = it.handler.getCoreContainer(); + final SolrParams params = it.req.getParams(); + + String commitName = params.required().get(CoreAdminParams.COMMIT_NAME); + String cname = params.required().get(CoreAdminParams.CORE); + try (SolrCore core = cc.getCore(cname)) { + if (core == null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to locate core " + cname); + } + + String indexDirPath = core.getIndexDir(); + IndexCommit ic = core.getDeletionPolicy().getLatestCommit(); + if (ic == null) { + RefCounted searcher = core.getSearcher(); + try { + ic = searcher.get().getIndexReader().getIndexCommit(); + } finally { + searcher.decref(); + } + } + SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager(); + mgr.snapshot(commitName, indexDirPath, ic.getGeneration()); + + it.rsp.add("core", core.getName()); + it.rsp.add("commitName", commitName); + it.rsp.add("indexDirPath", indexDirPath); + it.rsp.add("generation", ic.getGeneration()); + } + }), + DELETESNAPSHOT_OP(DELETESNAPSHOT, it -> { + CoreContainer cc = it.handler.getCoreContainer(); + final SolrParams params = it.req.getParams(); + + String commitName = params.required().get(CoreAdminParams.COMMIT_NAME); + String cname = params.required().get(CoreAdminParams.CORE); + try (SolrCore core = cc.getCore(cname)) { + if (core == null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to locate core " + cname); + } + + SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager(); + Optional metadata = mgr.release(commitName); + if (metadata.isPresent()) { + long gen = metadata.get().getGenerationNumber(); + String indexDirPath = metadata.get().getIndexDirPath(); + + // If the directory storing the snapshot is not the same as the *current* core + // index directory, then delete the files corresponding to this snapshot. + // Otherwise we leave the index files related to snapshot as is (assuming the + // underlying Solr IndexDeletionPolicy will clean them up appropriately). + if (!indexDirPath.equals(core.getIndexDir())) { + Directory d = core.getDirectoryFactory().get(indexDirPath, DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_NONE); + try { + SolrSnapshotManager.deleteIndexFiles(d, mgr.listSnapshotsInIndexDir(indexDirPath), gen); + } finally { + core.getDirectoryFactory().release(d); + } + } + } + } + }), + LISTSNAPSHOTS_OP(LISTSNAPSHOTS, it -> { + CoreContainer cc = it.handler.getCoreContainer(); + final SolrParams params = it.req.getParams(); + + String cname = params.required().get(CoreAdminParams.CORE); + try ( SolrCore core = cc.getCore(cname) ) { + if (core == null) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to locate core " + cname); + } + + SolrSnapshotMetaDataManager mgr = core.getSnapshotMetaDataManager(); + NamedList result = new NamedList(); + for (String name : mgr.listSnapshots()) { + Optional metadata = mgr.getSnapshotMetaData(name); + if ( metadata.isPresent() ) { + NamedList props = new NamedList<>(); + props.add("generation", String.valueOf(metadata.get().getGenerationNumber())); + props.add("indexDirPath", metadata.get().getIndexDirPath()); + result.add(name, props); + } + } + it.rsp.add("snapshots", result); + } }); final CoreAdminParams.CoreAdminAction action; diff --git a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java index 76b38fa3ebe..eb22d90ccfb 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java @@ -176,7 +176,7 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia query = rb.getQuery(); } else { try { - QParser parser = QParser.getParser(qs, null, req); + QParser parser = QParser.getParser(qs, req); query = parser.getQuery(); } catch (Exception e) { throw new IOException(e); @@ -198,7 +198,7 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia try { for (String fq : fqs) { if (fq != null && fq.trim().length() != 0 && !fq.equals("*:*")) { - QParser fqp = QParser.getParser(fq, null, req); + QParser fqp = QParser.getParser(fq, req); newFilters.add(fqp.getQuery()); } } diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index 931d3628464..0e3743991ee 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -202,7 +202,7 @@ public class QueryComponent extends SearchComponent filters = filters == null ? new ArrayList(fqs.length) : new ArrayList<>(filters); for (String fq : fqs) { if (fq != null && fq.trim().length()!=0) { - QParser fqp = QParser.getParser(fq, null, req); + QParser fqp = QParser.getParser(fq, req); filters.add(fqp.getQuery()); } } diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java index 9018a861eff..88dbc9d836b 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java @@ -158,7 +158,7 @@ public class RealTimeGetComponent extends SearchComponent filters = filters == null ? new ArrayList(fqs.length) : new ArrayList<>(filters); for (String fq : fqs) { if (fq != null && fq.trim().length()!=0) { - QParser fqp = QParser.getParser(fq, null, req); + QParser fqp = QParser.getParser(fq, req); filters.add(fqp.getQuery()); } } @@ -249,7 +249,8 @@ public class RealTimeGetComponent extends SearchComponent docid = segid + ctx.docBase; if (rb.getFilters() != null) { - for (Query q : rb.getFilters()) { + for (Query raw : rb.getFilters()) { + Query q = raw.rewrite(searcher.getIndexReader()); Scorer scorer = searcher.createWeight(q, false, 1f).scorer(ctx); if (scorer == null || segid != scorer.iterator().advance(segid)) { // filter doesn't match. @@ -448,7 +449,7 @@ public class RealTimeGetComponent extends SearchComponent ZkController zkController = rb.req.getCore().getCoreDescriptor().getCoreContainer().getZkController(); // if shards=... then use that - if (zkController != null && params.get("shards") == null) { + if (zkController != null && params.get(ShardParams.SHARDS) == null) { CloudDescriptor cloudDescriptor = rb.req.getCore().getCoreDescriptor().getCloudDescriptor(); String collection = cloudDescriptor.getCollectionName(); @@ -470,38 +471,46 @@ public class RealTimeGetComponent extends SearchComponent for (Map.Entry> entry : sliceToId.entrySet()) { String shard = entry.getKey(); - String shardIdList = StrUtils.join(entry.getValue(), ','); - ShardRequest sreq = new ShardRequest(); - - sreq.purpose = 1; + ShardRequest sreq = createShardRequest(rb, entry.getValue()); // sreq.shards = new String[]{shard}; // TODO: would be nice if this would work... sreq.shards = sliceToShards(rb, collection, shard); sreq.actualShards = sreq.shards; - sreq.params = new ModifiableSolrParams(); - sreq.params.set(ShardParams.SHARDS_QT,"/get"); // TODO: how to avoid hardcoding this and hit the same handler? - sreq.params.set("distrib",false); - sreq.params.set("ids", shardIdList); - + rb.addRequest(this, sreq); } } else { - String shardIdList = StrUtils.join(reqIds.allIds, ','); - ShardRequest sreq = new ShardRequest(); - - sreq.purpose = 1; + ShardRequest sreq = createShardRequest(rb, reqIds.allIds); sreq.shards = null; // ALL sreq.actualShards = sreq.shards; - sreq.params = new ModifiableSolrParams(); - sreq.params.set(ShardParams.SHARDS_QT,"/get"); // TODO: how to avoid hardcoding this and hit the same handler? - sreq.params.set("distrib",false); - sreq.params.set("ids", shardIdList); rb.addRequest(this, sreq); } return ResponseBuilder.STAGE_DONE; } + + /** + * Helper method for creating a new ShardRequest for the specified ids, based on the params + * specified for the current request. The new ShardRequest does not yet know anything about + * which shard/slice it will be sent to. + */ + private ShardRequest createShardRequest(final ResponseBuilder rb, final List ids) { + final ShardRequest sreq = new ShardRequest(); + sreq.purpose = 1; + sreq.params = new ModifiableSolrParams(rb.req.getParams()); + + // TODO: how to avoid hardcoding this and hit the same handler? + sreq.params.set(ShardParams.SHARDS_QT,"/get"); + sreq.params.set("distrib",false); + + sreq.params.remove(ShardParams.SHARDS); + sreq.params.remove("id"); + sreq.params.remove("ids"); + sreq.params.set("ids", StrUtils.join(ids, ',')); + + return sreq; + } private String[] sliceToShards(ResponseBuilder rb, String collection, String slice) { String lookup = collection + '_' + slice; // seems either form may be filled in rb.slices? diff --git a/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java b/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java index 2e1bebe8b25..9b510fed665 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java @@ -60,7 +60,6 @@ import org.apache.solr.schema.FieldType; import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.DocSet; import org.apache.solr.search.QParser; -import org.apache.solr.search.QParserPlugin; import org.apache.solr.search.SyntaxError; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.spelling.AbstractLuceneSpellChecker; @@ -242,7 +241,7 @@ public class SpellCheckComponent extends SearchComponent implements SolrCoreAwar try { if (maxResultsFilterQueryString != null) { // Get the default Lucene query parser - QParser parser = QParser.getParser(maxResultsFilterQueryString, QParserPlugin.DEFAULT_QTYPE, rb.req); + QParser parser = QParser.getParser(maxResultsFilterQueryString, rb.req); DocSet s = searcher.getDocSet(parser.getQuery()); maxResultsByFilters = s.size(); } else { diff --git a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java index 017deb4f617..c2f68f93bf8 100644 --- a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java +++ b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java @@ -321,7 +321,7 @@ public class SimpleFacets { public void getFacetQueryCount(ParsedParams parsed, NamedList res) throws SyntaxError, IOException { // TODO: slight optimization would prevent double-parsing of any localParams // TODO: SOLR-7753 - Query qobj = QParser.getParser(parsed.facetValue, null, req).getQuery(); + Query qobj = QParser.getParser(parsed.facetValue, req).getQuery(); if (qobj == null) { res.add(parsed.key, 0); diff --git a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java index 973151f9e60..e829e037c2d 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java @@ -77,7 +77,7 @@ public class ChildDocTransformerFactory extends TransformerFactory { BitSetProducer parentsFilter = null; try { - Query parentFilterQuery = QParser.getParser( parentFilter, null, req).getQuery(); + Query parentFilterQuery = QParser.getParser( parentFilter, req).getQuery(); parentsFilter = new QueryBitSetProducer(new QueryWrapperFilter(parentFilterQuery)); } catch (SyntaxError syntaxError) { throw new SolrException( ErrorCode.BAD_REQUEST, "Failed to create correct parent filter query" ); @@ -86,7 +86,7 @@ public class ChildDocTransformerFactory extends TransformerFactory { Query childFilterQuery = null; if(childFilter != null) { try { - childFilterQuery = QParser.getParser( childFilter, null, req).getQuery(); + childFilterQuery = QParser.getParser( childFilter, req).getQuery(); } catch (SyntaxError syntaxError) { throw new SolrException( ErrorCode.BAD_REQUEST, "Failed to create correct child filter query" ); } diff --git a/solr/core/src/java/org/apache/solr/response/transform/SubQueryAugmenterFactory.java b/solr/core/src/java/org/apache/solr/response/transform/SubQueryAugmenterFactory.java index 40cc31365e5..cbe69985bca 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/SubQueryAugmenterFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/SubQueryAugmenterFactory.java @@ -76,6 +76,15 @@ import org.apache.solr.search.TermsQParserPlugin; * its' native parameters like collection, shards for subquery, eg
* q=*:*&fl=*,foo:[subquery]&foo.q=cloud&foo.collection=departments * + *

When used in Real Time Get

+ *

+ * When used in the context of a Real Time Get, the values from each document that are used + * in the qubquery are the "real time" values (possibly from the transaction log), but the query + * itself is still executed against the currently open searcher. Note that this means if a + * document is updated but not yet committed, an RTG request for that document that uses + * [subquery] could include the older (committed) version of that document, + * with differnet field values, in the subquery results. + *

*/ public class SubQueryAugmenterFactory extends TransformerFactory{ @@ -303,6 +312,14 @@ class SubQueryAugmenter extends DocTransformer { public String getName() { return name; } + + /** + * Returns false -- this transformer does use an IndexSearcher, but it does not (neccessarily) need + * the searcher from the ResultContext of the document being returned. Instead we use the current + * "live" searcher for the specified core. + */ + @Override + public boolean needsSolrIndexSearcher() { return false; } @Override public void transform(SolrDocument doc, int docid, float score) { diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index 43d681425f7..c48518ca583 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -1500,10 +1500,12 @@ public class IndexSchema { (v1, v2) -> v2, LinkedHashMap::new)); } - public static Map nameMapping = Collections.unmodifiableMap(Stream.of(Handler.values()) - .collect(Collectors.toMap(Handler::getNameLower , Handler::getRealName))); } + public static Map nameMapping = Collections.unmodifiableMap(Stream.of(SchemaProps.Handler.values()) + .collect(Collectors.toMap(SchemaProps.Handler::getNameLower , SchemaProps.Handler::getRealName))); + + public Map getNamedPropertyValues(String name, SolrParams params) { return new SchemaProps(name, params, this).toMap(); diff --git a/solr/core/src/java/org/apache/solr/search/CacheConfig.java b/solr/core/src/java/org/apache/solr/search/CacheConfig.java index 40e54dccb4e..ee333f8787b 100644 --- a/solr/core/src/java/org/apache/solr/search/CacheConfig.java +++ b/solr/core/src/java/org/apache/solr/search/CacheConfig.java @@ -17,10 +17,10 @@ package org.apache.solr.search; import javax.xml.xpath.XPathConstants; - import java.lang.invoke.MethodHandles; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -75,14 +75,15 @@ public class CacheConfig implements MapSerializable{ this.regenerator = regenerator; } - public static CacheConfig[] getMultipleConfigs(SolrConfig solrConfig, String configPath) { - NodeList nodes = (NodeList)solrConfig.evaluate(configPath, XPathConstants.NODESET); - if (nodes==null || nodes.getLength()==0) return null; - CacheConfig[] configs = new CacheConfig[nodes.getLength()]; - for (int i=0; i getMultipleConfigs(SolrConfig solrConfig, String configPath) { + NodeList nodes = (NodeList) solrConfig.evaluate(configPath, XPathConstants.NODESET); + if (nodes == null || nodes.getLength() == 0) return new LinkedHashMap<>(); + Map result = new HashMap<>(nodes.getLength()); + for (int i = 0; i < nodes.getLength(); i++) { + CacheConfig config = getConfig(solrConfig, nodes.item(i).getNodeName(), DOMUtil.toMap(nodes.item(i).getAttributes()), configPath); + result.put(config.args.get(NAME), config); } - return configs; + return result; } @@ -101,9 +102,14 @@ public class CacheConfig implements MapSerializable{ public static CacheConfig getConfig(SolrConfig solrConfig, String nodeName, Map attrs, String xpath) { CacheConfig config = new CacheConfig(); config.nodeName = nodeName; + Map attrsCopy = new LinkedHashMap<>(attrs.size()); + for (Map.Entry e : attrs.entrySet()) { + attrsCopy.put(e.getKey(), String.valueOf(e.getValue())); + } + attrs = attrsCopy; config.args = attrs; - Map map = solrConfig.getOverlay().getEditableSubProperties(xpath); + Map map = xpath == null ? null : solrConfig.getOverlay().getEditableSubProperties(xpath); if(map != null){ HashMap mapCopy = new HashMap<>(config.args); for (Map.Entry e : map.entrySet()) { diff --git a/solr/core/src/java/org/apache/solr/search/Grouping.java b/solr/core/src/java/org/apache/solr/search/Grouping.java index 1ebf574d6b7..80a6aebda85 100644 --- a/solr/core/src/java/org/apache/solr/search/Grouping.java +++ b/solr/core/src/java/org/apache/solr/search/Grouping.java @@ -223,7 +223,7 @@ public class Grouping { } public void addQueryCommand(String groupByStr, SolrQueryRequest request) throws SyntaxError { - QParser parser = QParser.getParser(groupByStr, null, request); + QParser parser = QParser.getParser(groupByStr, request); Query gq = parser.getQuery(); Grouping.CommandQuery gc = new CommandQuery(); gc.query = gq; diff --git a/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java new file mode 100644 index 00000000000..ee8cf67a6e4 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search; + + +import java.io.IOException; +import java.util.TreeSet; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.SparseFixedBitSet; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.request.SolrQueryRequest; + +public class IGainTermsQParserPlugin extends QParserPlugin { + + public static final String NAME = "igain"; + + @Override + public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + return new IGainTermsQParser(qstr, localParams, params, req); + } + + private static class IGainTermsQParser extends QParser { + + public IGainTermsQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + super(qstr, localParams, params, req); + } + + @Override + public Query parse() throws SyntaxError { + + String field = getParam("field"); + String outcome = getParam("outcome"); + int numTerms = Integer.parseInt(getParam("numTerms")); + int positiveLabel = Integer.parseInt(getParam("positiveLabel")); + + return new IGainTermsQuery(field, outcome, positiveLabel, numTerms); + } + } + + private static class IGainTermsQuery extends AnalyticsQuery { + + private String field; + private String outcome; + private int numTerms; + private int positiveLabel; + + public IGainTermsQuery(String field, String outcome, int positiveLabel, int numTerms) { + this.field = field; + this.outcome = outcome; + this.numTerms = numTerms; + this.positiveLabel = positiveLabel; + } + + @Override + public DelegatingCollector getAnalyticsCollector(ResponseBuilder rb, IndexSearcher searcher) { + return new IGainTermsCollector(rb, searcher, field, outcome, positiveLabel, numTerms); + } + } + + private static class IGainTermsCollector extends DelegatingCollector { + + private String field; + private String outcome; + private IndexSearcher searcher; + private ResponseBuilder rb; + private int positiveLabel; + private int numTerms; + private int count; + + private NumericDocValues leafOutcomeValue; + private SparseFixedBitSet positiveSet; + private SparseFixedBitSet negativeSet; + + + private int numPositiveDocs; + + + public IGainTermsCollector(ResponseBuilder rb, IndexSearcher searcher, String field, String outcome, int positiveLabel, int numTerms) { + this.rb = rb; + this.searcher = searcher; + this.field = field; + this.outcome = outcome; + this.positiveSet = new SparseFixedBitSet(searcher.getIndexReader().maxDoc()); + this.negativeSet = new SparseFixedBitSet(searcher.getIndexReader().maxDoc()); + + this.numTerms = numTerms; + this.positiveLabel = positiveLabel; + } + + @Override + protected void doSetNextReader(LeafReaderContext context) throws IOException { + super.doSetNextReader(context); + LeafReader reader = context.reader(); + leafOutcomeValue = reader.getNumericDocValues(outcome); + } + + @Override + public void collect(int doc) throws IOException { + super.collect(doc); + ++count; + if (leafOutcomeValue.get(doc) == positiveLabel) { + positiveSet.set(context.docBase + doc); + numPositiveDocs++; + } else { + negativeSet.set(context.docBase + doc); + } + } + + @Override + public void finish() throws IOException { + NamedList analytics = new NamedList(); + NamedList topFreq = new NamedList(); + + NamedList allFreq = new NamedList(); + + rb.rsp.add("featuredTerms", analytics); + rb.rsp.add("docFreq", topFreq); + rb.rsp.add("numDocs", count); + + TreeSet topTerms = new TreeSet<>(); + + double numDocs = count; + double pc = numPositiveDocs / numDocs; + double entropyC = binaryEntropy(pc); + + Terms terms = MultiFields.getFields(searcher.getIndexReader()).terms(field); + TermsEnum termsEnum = terms.iterator(); + BytesRef term; + PostingsEnum postingsEnum = null; + while ((term = termsEnum.next()) != null) { + postingsEnum = termsEnum.postings(postingsEnum); + int xc = 0; + int nc = 0; + while (postingsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { + if (positiveSet.get(postingsEnum.docID())) { + xc++; + } else if (negativeSet.get(postingsEnum.docID())) { + nc++; + } + } + + int docFreq = xc+nc; + + double entropyContainsTerm = binaryEntropy( (double) xc / docFreq ); + double entropyNotContainsTerm = binaryEntropy( (double) (numPositiveDocs - xc) / (numDocs - docFreq + 1) ); + double score = entropyC - ( (docFreq / numDocs) * entropyContainsTerm + (1.0 - docFreq / numDocs) * entropyNotContainsTerm); + + topFreq.add(term.utf8ToString(), docFreq); + if (topTerms.size() < numTerms) { + topTerms.add(new TermWithScore(term.utf8ToString(), score)); + } else { + if (topTerms.first().score < score) { + topTerms.pollFirst(); + topTerms.add(new TermWithScore(term.utf8ToString(), score)); + } + } + } + + for (TermWithScore topTerm : topTerms) { + analytics.add(topTerm.term, topTerm.score); + topFreq.add(topTerm.term, allFreq.get(topTerm.term)); + } + + if (this.delegate instanceof DelegatingCollector) { + ((DelegatingCollector) this.delegate).finish(); + } + } + + private double binaryEntropy(double prob) { + if (prob == 0 || prob == 1) return 0; + return (-1 * prob * Math.log(prob)) + (-1 * (1.0 - prob) * Math.log(1.0 - prob)); + } + + } + + + + private static class TermWithScore implements Comparable{ + public final String term; + public final double score; + + public TermWithScore(String term, double score) { + this.term = term; + this.score = score; + } + + @Override + public int hashCode() { + return term.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) return false; + if (obj.getClass() != getClass()) return false; + TermWithScore other = (TermWithScore) obj; + return other.term.equals(this.term); + } + + @Override + public int compareTo(TermWithScore o) { + int cmp = Double.compare(this.score, o.score); + if (cmp == 0) { + return this.term.compareTo(o.term); + } else { + return cmp; + } + } + } +} + + diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java index de96c96464e..e0bb09bf518 100644 --- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -98,7 +98,7 @@ public class JoinQParserPlugin extends QParserPlugin { RefCounted fromHolder = null; LocalSolrQueryRequest otherReq = new LocalSolrQueryRequest(fromCore, params); try { - QParser parser = QParser.getParser(v, "lucene", otherReq); + QParser parser = QParser.getParser(v, otherReq); fromQuery = parser.getQuery(); fromHolder = fromCore.getRegisteredSearcher(); if (fromHolder != null) fromCoreOpenTime = fromHolder.get().getOpenNanoTime(); diff --git a/solr/core/src/java/org/apache/solr/search/LRUCache.java b/solr/core/src/java/org/apache/solr/search/LRUCache.java index 8e030fd2d43..0d9f40665d6 100644 --- a/solr/core/src/java/org/apache/solr/search/LRUCache.java +++ b/solr/core/src/java/org/apache/solr/search/LRUCache.java @@ -23,7 +23,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountables; @@ -61,11 +61,11 @@ public class LRUCache extends SolrCacheBase implements SolrCache, Acco * of an LRUCache at the same time. Make sure everything is thread safe. */ private static class CumulativeStats { - AtomicLong lookups = new AtomicLong(); - AtomicLong hits = new AtomicLong(); - AtomicLong inserts = new AtomicLong(); - AtomicLong evictions = new AtomicLong(); - AtomicLong evictionsRamUsage = new AtomicLong(); + LongAdder lookups = new LongAdder(); + LongAdder hits = new LongAdder(); + LongAdder inserts = new LongAdder(); + LongAdder evictions = new LongAdder(); + LongAdder evictionsRamUsage = new LongAdder(); } private CumulativeStats stats; @@ -124,8 +124,8 @@ public class LRUCache extends SolrCacheBase implements SolrCache, Acco iterator.remove(); evictions++; evictionsRamUsage++; - stats.evictions.incrementAndGet(); - stats.evictionsRamUsage.incrementAndGet(); + stats.evictions.increment(); + stats.evictionsRamUsage.increment(); } while (iterator.hasNext() && ramBytesUsed > maxRamBytes); // must return false according to javadocs of removeEldestEntry if we're modifying // the map ourselves @@ -135,7 +135,7 @@ public class LRUCache extends SolrCacheBase implements SolrCache, Acco // this doesn't need to be synchronized because it will // only be called in the context of a higher level synchronized block. evictions++; - stats.evictions.incrementAndGet(); + stats.evictions.increment(); return true; } } @@ -180,7 +180,7 @@ public class LRUCache extends SolrCacheBase implements SolrCache, Acco public V put(K key, V value) { synchronized (map) { if (getState() == State.LIVE) { - stats.inserts.incrementAndGet(); + stats.inserts.increment(); } // increment local inserts regardless of state??? @@ -232,10 +232,10 @@ public class LRUCache extends SolrCacheBase implements SolrCache, Acco if (getState() == State.LIVE) { // only increment lookups and hits if we are live. lookups++; - stats.lookups.incrementAndGet(); + stats.lookups.increment(); if (val!=null) { hits++; - stats.hits.incrementAndGet(); + stats.hits.increment(); } } return val; @@ -341,15 +341,15 @@ public class LRUCache extends SolrCacheBase implements SolrCache, Acco } lst.add("warmupTime", warmupTime); - long clookups = stats.lookups.get(); - long chits = stats.hits.get(); + long clookups = stats.lookups.longValue(); + long chits = stats.hits.longValue(); lst.add("cumulative_lookups", clookups); lst.add("cumulative_hits", chits); lst.add("cumulative_hitratio", calcHitRatio(clookups, chits)); - lst.add("cumulative_inserts", stats.inserts.get()); - lst.add("cumulative_evictions", stats.evictions.get()); + lst.add("cumulative_inserts", stats.inserts.longValue()); + lst.add("cumulative_evictions", stats.evictions.longValue()); if (maxRamBytes != Long.MAX_VALUE) { - lst.add("cumulative_evictionsRamUsage", stats.evictionsRamUsage.get()); + lst.add("cumulative_evictionsRamUsage", stats.evictionsRamUsage.longValue()); } return lst; diff --git a/solr/core/src/java/org/apache/solr/search/QParser.java b/solr/core/src/java/org/apache/solr/search/QParser.java index 45b698e7ea5..898fdf1bc7c 100644 --- a/solr/core/src/java/org/apache/solr/search/QParser.java +++ b/solr/core/src/java/org/apache/solr/search/QParser.java @@ -263,6 +263,17 @@ public abstract class QParser { debugInfo.add("QParser", this.getClass().getSimpleName()); } + /** Create a QParser to parse qstr, + * using the "lucene" (QParserPlugin.DEFAULT_QTYPE) query parser. + * The query parser may be overridden by local parameters in the query + * string itself. For example if + * qstr={!prefix f=myfield}foo + * then the prefix query parser will be used. + */ + public static QParser getParser(String qstr, SolrQueryRequest req) throws SyntaxError { + return getParser(qstr, QParserPlugin.DEFAULT_QTYPE, req); + } + /** Create a QParser to parse qstr, * assuming that the default query parser is defaultParser. * The query parser may be overridden by local parameters in the query diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java index 7a6247f287c..573286b91be 100644 --- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java @@ -16,6 +16,11 @@ */ package org.apache.solr.search; +import java.net.URL; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrInfoMBean; @@ -26,11 +31,6 @@ import org.apache.solr.search.join.GraphQParserPlugin; import org.apache.solr.search.mlt.MLTQParserPlugin; import org.apache.solr.util.plugin.NamedListInitializedPlugin; -import java.net.URL; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrInfoMBean { /** internal use - name of the default parser */ public static final String DEFAULT_QTYPE = LuceneQParserPlugin.NAME; @@ -77,6 +77,8 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI map.put(GraphQParserPlugin.NAME, GraphQParserPlugin.class); map.put(XmlQParserPlugin.NAME, XmlQParserPlugin.class); map.put(GraphTermsQParserPlugin.NAME, GraphTermsQParserPlugin.class); + map.put(IGainTermsQParserPlugin.NAME, IGainTermsQParserPlugin.class); + map.put(TextLogisticRegressionQParserPlugin.NAME, TextLogisticRegressionQParserPlugin.class); standardPlugins = Collections.unmodifiableMap(map); } diff --git a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java index 37919f45136..a903968058a 100644 --- a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java @@ -32,6 +32,7 @@ import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryRescorer; +import org.apache.lucene.search.Rescorer; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TopDocs; @@ -41,7 +42,6 @@ import org.apache.lucene.search.TopScoreDocCollector; import org.apache.lucene.search.Weight; import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.handler.component.MergeStrategy; import org.apache.solr.handler.component.QueryElevationComponent; @@ -82,7 +82,7 @@ public class ReRankQParserPlugin extends QParserPlugin { if (reRankQueryString == null || reRankQueryString.trim().length() == 0) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, RERANK_QUERY+" parameter is mandatory"); } - QParser reRankParser = QParser.getParser(reRankQueryString, null, req); + QParser reRankParser = QParser.getParser(reRankQueryString, req); Query reRankQuery = reRankParser.parse(); int reRankDocs = localParams.getInt(RERANK_DOCS, RERANK_DOCS_DEFAULT); @@ -90,19 +90,35 @@ public class ReRankQParserPlugin extends QParserPlugin { double reRankWeight = localParams.getDouble(RERANK_WEIGHT, RERANK_WEIGHT_DEFAULT); - int start = params.getInt(CommonParams.START,CommonParams.START_DEFAULT); - int rows = params.getInt(CommonParams.ROWS,CommonParams.ROWS_DEFAULT); - int length = start+rows; - return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight, length); + return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight); + } + } + + private final class ReRankQueryRescorer extends QueryRescorer { + + final double reRankWeight; + + public ReRankQueryRescorer(Query reRankQuery, double reRankWeight) { + super(reRankQuery); + this.reRankWeight = reRankWeight; + } + + @Override + protected float combine(float firstPassScore, boolean secondPassMatches, float secondPassScore) { + float score = firstPassScore; + if (secondPassMatches) { + score += reRankWeight * secondPassScore; + } + return score; } } private final class ReRankQuery extends RankQuery { private Query mainQuery = defaultQuery; - private Query reRankQuery; - private int reRankDocs; - private int length; - private double reRankWeight; + final private Query reRankQuery; + final private int reRankDocs; + final private double reRankWeight; + final private Rescorer reRankQueryRescorer; private Map boostedPriority; public int hashCode() { @@ -121,11 +137,11 @@ public class ReRankQParserPlugin extends QParserPlugin { reRankDocs == rrq.reRankDocs; } - public ReRankQuery(Query reRankQuery, int reRankDocs, double reRankWeight, int length) { + public ReRankQuery(Query reRankQuery, int reRankDocs, double reRankWeight) { this.reRankQuery = reRankQuery; this.reRankDocs = reRankDocs; this.reRankWeight = reRankWeight; - this.length = length; + this.reRankQueryRescorer = new ReRankQueryRescorer(reRankQuery, reRankWeight); } public RankQuery wrap(Query _mainQuery) { @@ -149,7 +165,7 @@ public class ReRankQParserPlugin extends QParserPlugin { } } - return new ReRankCollector(reRankDocs, length, reRankQuery, reRankWeight, cmd, searcher, boostedPriority); + return new ReRankCollector(reRankDocs, len, reRankQueryRescorer, cmd, searcher, boostedPriority); } @Override @@ -166,63 +182,49 @@ public class ReRankQParserPlugin extends QParserPlugin { public Query rewrite(IndexReader reader) throws IOException { Query q = mainQuery.rewrite(reader); if (q != mainQuery) { - return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight, length).wrap(q); + return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight).wrap(q); } return super.rewrite(reader); } public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException{ - return new ReRankWeight(mainQuery, reRankQuery, reRankWeight, searcher, needsScores, boost); + return new ReRankWeight(mainQuery, reRankQueryRescorer, searcher, needsScores, boost); } } private class ReRankWeight extends FilterWeight { - private Query reRankQuery; private IndexSearcher searcher; - private double reRankWeight; + final private Rescorer reRankQueryRescorer; - public ReRankWeight(Query mainQuery, Query reRankQuery, double reRankWeight, IndexSearcher searcher, boolean needsScores, float boost) throws IOException { + public ReRankWeight(Query mainQuery, Rescorer reRankQueryRescorer, IndexSearcher searcher, boolean needsScores, float boost) throws IOException { super(mainQuery, mainQuery.createWeight(searcher, needsScores, boost)); - this.reRankQuery = reRankQuery; this.searcher = searcher; - this.reRankWeight = reRankWeight; + this.reRankQueryRescorer = reRankQueryRescorer; } public Explanation explain(LeafReaderContext context, int doc) throws IOException { Explanation mainExplain = in.explain(context, doc); - return new QueryRescorer(reRankQuery) { - @Override - protected float combine(float firstPassScore, boolean secondPassMatches, float secondPassScore) { - float score = firstPassScore; - if (secondPassMatches) { - score += reRankWeight * secondPassScore; - } - return score; - } - }.explain(searcher, mainExplain, context.docBase+doc); + return reRankQueryRescorer.explain(searcher, mainExplain, context.docBase+doc); } } private class ReRankCollector extends TopDocsCollector { - private Query reRankQuery; - private TopDocsCollector mainCollector; - private IndexSearcher searcher; - private int reRankDocs; - private int length; - private double reRankWeight; - private Map boostedPriority; + final private TopDocsCollector mainCollector; + final private IndexSearcher searcher; + final private int reRankDocs; + final private int length; + final private Map boostedPriority; + final private Rescorer reRankQueryRescorer; public ReRankCollector(int reRankDocs, int length, - Query reRankQuery, - double reRankWeight, + Rescorer reRankQueryRescorer, QueryCommand cmd, IndexSearcher searcher, Map boostedPriority) throws IOException { super(null); - this.reRankQuery = reRankQuery; this.reRankDocs = reRankDocs; this.length = length; this.boostedPriority = boostedPriority; @@ -234,7 +236,7 @@ public class ReRankQParserPlugin extends QParserPlugin { this.mainCollector = TopFieldCollector.create(sort, Math.max(this.reRankDocs, length), false, true, true); } this.searcher = searcher; - this.reRankWeight = reRankWeight; + this.reRankQueryRescorer = reRankQueryRescorer; } public int getTotalHits() { @@ -276,16 +278,8 @@ public class ReRankQParserPlugin extends QParserPlugin { mainDocs.scoreDocs = reRankScoreDocs; - TopDocs rescoredDocs = new QueryRescorer(reRankQuery) { - @Override - protected float combine(float firstPassScore, boolean secondPassMatches, float secondPassScore) { - float score = firstPassScore; - if (secondPassMatches) { - score += reRankWeight * secondPassScore; - } - return score; - } - }.rescore(searcher, mainDocs, mainDocs.scoreDocs.length); + TopDocs rescoredDocs = reRankQueryRescorer + .rescore(searcher, mainDocs, mainDocs.scoreDocs.length); Arrays.sort(rescoredDocs.scoreDocs, new BoostedComp(boostedDocs, mainDocs.scoreDocs, rescoredDocs.getMaxScore())); @@ -325,16 +319,8 @@ public class ReRankQParserPlugin extends QParserPlugin { mainDocs.scoreDocs = reRankScoreDocs; - TopDocs rescoredDocs = new QueryRescorer(reRankQuery) { - @Override - protected float combine(float firstPassScore, boolean secondPassMatches, float secondPassScore) { - float score = firstPassScore; - if (secondPassMatches) { - score += reRankWeight * secondPassScore; - } - return score; - } - }.rescore(searcher, mainDocs, mainDocs.scoreDocs.length); + TopDocs rescoredDocs = reRankQueryRescorer + .rescore(searcher, mainDocs, mainDocs.scoreDocs.length); //Lower howMany to return if we've collected fewer documents. howMany = Math.min(howMany, mainScoreDocs.length); diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index cc719f0be37..0f480c67579 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -36,62 +36,16 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import com.google.common.base.Function; +import com.google.common.base.Objects; +import com.google.common.collect.Iterables; import org.apache.lucene.document.Document; import org.apache.lucene.document.DocumentStoredFieldVisitor; import org.apache.lucene.document.LazyDocument; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.ExitableDirectoryReader; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiPostingsEnum; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.*; import org.apache.lucene.index.StoredFieldVisitor.Status; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermContext; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CollectionStatistics; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.EarlyTerminatingSortingCollector; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.MultiCollector; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.SimpleCollector; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TermStatistics; -import org.apache.lucene.search.TimeLimitingCollector; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopDocsCollector; -import org.apache.lucene.search.TopFieldCollector; -import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.search.TopScoreDocCollector; -import org.apache.lucene.search.TotalHitCountCollector; -import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -128,10 +82,6 @@ import org.apache.solr.update.SolrIndexConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Function; -import com.google.common.base.Objects; -import com.google.common.collect.Iterables; - /** * SolrIndexSearcher adds schema awareness and caching functionality over {@link IndexSearcher}. * @@ -337,13 +287,12 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI documentCache = solrConfig.documentCacheConfig == null ? null : solrConfig.documentCacheConfig.newInstance(); if (documentCache != null) clist.add(documentCache); - if (solrConfig.userCacheConfigs == null) { + if (solrConfig.userCacheConfigs.isEmpty()) { cacheMap = NO_GENERIC_CACHES; } else { - cacheMap = new HashMap<>(solrConfig.userCacheConfigs.length); - for (CacheConfig userCacheConfig : solrConfig.userCacheConfigs) { - SolrCache cache = null; - if (userCacheConfig != null) cache = userCacheConfig.newInstance(); + cacheMap = new HashMap<>(solrConfig.userCacheConfigs.size()); + for (Map.Entry e : solrConfig.userCacheConfigs.entrySet()) { + SolrCache cache = e.getValue().newInstance(); if (cache != null) { cacheMap.put(cache.name(), cache); clist.add(cache); diff --git a/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java new file mode 100644 index 00000000000..0ca9f723f32 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.SparseFixedBitSet; +import org.apache.solr.client.solrj.io.ClassificationEvaluation; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.request.SolrQueryRequest; + +/** + * Returns an AnalyticsQuery implementation that performs + * one Gradient Descent iteration of a result set to train a + * logistic regression model + * + * The TextLogitStream provides the parallel iterative framework for this class. + **/ + +public class TextLogisticRegressionQParserPlugin extends QParserPlugin { + public static final String NAME = "tlogit"; + + @Override + public void init(NamedList args) { + } + + @Override + public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + return new TextLogisticRegressionQParser(qstr, localParams, params, req); + } + + private static class TextLogisticRegressionQParser extends QParser{ + + TextLogisticRegressionQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + super(qstr, localParams, params, req); + } + + public Query parse() { + + String fs = params.get("feature"); + String[] terms = params.get("terms").split(","); + String ws = params.get("weights"); + String dfsStr = params.get("idfs"); + int iteration = params.getInt("iteration"); + String outcome = params.get("outcome"); + int positiveLabel = params.getInt("positiveLabel", 1); + double threshold = params.getDouble("threshold", 0.5); + double alpha = params.getDouble("alpha", 0.01); + + double[] idfs = new double[terms.length]; + String[] idfsArr = dfsStr.split(","); + for (int i = 0; i < idfsArr.length; i++) { + idfs[i] = Double.parseDouble(idfsArr[i]); + } + + double[] weights = new double[terms.length+1]; + + if(ws != null) { + String[] wa = ws.split(","); + for (int i = 0; i < wa.length; i++) { + weights[i] = Double.parseDouble(wa[i]); + } + } else { + for(int i=0; i docVectors = new HashMap<>(); + Terms terms = MultiFields.getFields(searcher.getIndexReader()).terms(trainingParams.feature); + TermsEnum termsEnum = terms.iterator(); + PostingsEnum postingsEnum = null; + int termIndex = 0; + for (String termStr : trainingParams.terms) { + BytesRef term = new BytesRef(termStr); + if (termsEnum.seekExact(term)) { + postingsEnum = termsEnum.postings(postingsEnum); + while (postingsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { + int docId = postingsEnum.docID(); + if (docsSet.get(docId)) { + double[] vector = docVectors.get(docId); + if (vector == null) { + vector = new double[trainingParams.terms.length+1]; + vector[0] = 1.0; + docVectors.put(docId, vector); + } + vector[termIndex + 1] = trainingParams.idfs[termIndex] * (1.0 + Math.log(postingsEnum.freq())); + } + } + } + termIndex++; + } + + for (Map.Entry entry : docVectors.entrySet()) { + double[] vector = entry.getValue(); + int outcome = 0; + if (positiveDocsSet.get(entry.getKey())) { + outcome = 1; + } + double sig = sigmoid(sum(multiply(vector, weights))); + double error = sig - outcome; + double lastSig = sigmoid(sum(multiply(vector, trainingParams.weights))); + totalError += Math.abs(lastSig - outcome); + classificationEvaluation.count(outcome, lastSig >= trainingParams.threshold ? 1 : 0); + + workingDeltas = multiply(error * trainingParams.alpha, vector); + + for(int i = 0; i< workingDeltas.length; i++) { + weights[i] -= workingDeltas[i]; + } + } + + NamedList analytics = new NamedList(); + rbsp.rsp.add("logit", analytics); + + List outWeights = new ArrayList<>(); + for(Double d : weights) { + outWeights.add(d); + } + + analytics.add("weights", outWeights); + analytics.add("error", totalError); + analytics.add("evaluation", classificationEvaluation.toMap()); + analytics.add("feature", trainingParams.feature); + analytics.add("positiveLabel", trainingParams.positiveLabel); + if(this.delegate instanceof DelegatingCollector) { + ((DelegatingCollector)this.delegate).finish(); + } + } + + private double sigmoid(double in) { + double d = 1.0 / (1+Math.exp(-in)); + return d; + } + + private double[] multiply(double[] vals, double[] weights) { + for(int i = 0; i < vals.length; ++i) { + workingDeltas[i] = vals[i] * weights[i]; + } + + return workingDeltas; + } + + private double[] multiply(double d, double[] vals) { + for(int i = 0; i cmp > 0); + } + }); + + addParser("lt", new ValueSourceParser() { + @Override + public ValueSource parse(FunctionQParser fp) throws SyntaxError { + ValueSource lhsValSource = fp.parseValueSource(); + ValueSource rhsValSource = fp.parseValueSource(); + + return new SolrComparisonBoolFunction(lhsValSource, rhsValSource, "lt", (cmp) -> cmp < 0); + } + }); + + addParser("gte", new ValueSourceParser() { + @Override + public ValueSource parse(FunctionQParser fp) throws SyntaxError { + ValueSource lhsValSource = fp.parseValueSource(); + ValueSource rhsValSource = fp.parseValueSource(); + + return new SolrComparisonBoolFunction(lhsValSource, rhsValSource, "gte", (cmp) -> cmp >= 0); + + } + }); + + addParser("lte", new ValueSourceParser() { + @Override + public ValueSource parse(FunctionQParser fp) throws SyntaxError { + ValueSource lhsValSource = fp.parseValueSource(); + ValueSource rhsValSource = fp.parseValueSource(); + + return new SolrComparisonBoolFunction(lhsValSource, rhsValSource, "lte", (cmp) -> cmp <= 0); + } + }); + + addParser("eq", new ValueSourceParser() { + @Override + public ValueSource parse(FunctionQParser fp) throws SyntaxError { + ValueSource lhsValSource = fp.parseValueSource(); + ValueSource rhsValSource = fp.parseValueSource(); + + return new SolrComparisonBoolFunction(lhsValSource, rhsValSource, "eq", (cmp) -> cmp == 0); + } + }); + addParser("def", new ValueSourceParser() { @Override public ValueSource parse(FunctionQParser fp) throws SyntaxError { diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java index 0e7e82a0e50..a5ec1dbab91 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java @@ -839,9 +839,13 @@ class FacetFieldProcessorStream extends FacetFieldProcessor implements Closeable createAccs(-1, 1); // Minimum term docFreq in order to use the filterCache for that term. - int defaultMinDf = Math.max(fcontext.searcher.maxDoc() >> 4, 3); // (minimum of 3 is for test coverage purposes) - int minDfFilterCache = freq.cacheDf == 0 ? defaultMinDf : freq.cacheDf; - if (minDfFilterCache == -1) minDfFilterCache = Integer.MAX_VALUE; // -1 means never cache + if (freq.cacheDf == -1) { // -1 means never cache + minDfFilterCache = Integer.MAX_VALUE; + } else if (freq.cacheDf == 0) { // default; compute as fraction of maxDoc + minDfFilterCache = Math.max(fcontext.searcher.maxDoc() >> 4, 3); // (minimum of 3 is for test coverage purposes) + } else { + minDfFilterCache = freq.cacheDf; + } docs = fcontext.base; fastForRandomSet = null; diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java index 37013b0a330..b1281f4b23b 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetProcessor.java @@ -96,7 +96,7 @@ public class FacetProcessor { String parentStr = freq.domain.parents; Query parentQuery; try { - QParser parser = QParser.getParser(parentStr, null, fcontext.req); + QParser parser = QParser.getParser(parentStr, fcontext.req); parentQuery = parser.getQuery(); } catch (SyntaxError err) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing block join parent specification: " + parentStr); diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java index 4c14ab8ad2c..0446202022a 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java @@ -478,7 +478,7 @@ class FacetQueryParser extends FacetParser { // TODO: substats that are from defaults!!! if (qstring != null) { - QParser parser = QParser.getParser(qstring, null, getSolrRequest()); + QParser parser = QParser.getParser(qstring, getSolrRequest()); facet.q = parser.getQuery(); } diff --git a/solr/core/src/java/org/apache/solr/search/function/SolrComparisonBoolFunction.java b/solr/core/src/java/org/apache/solr/search/function/SolrComparisonBoolFunction.java new file mode 100644 index 00000000000..c994fbbbc17 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/function/SolrComparisonBoolFunction.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.function; + +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.docvalues.IntDocValues; +import org.apache.lucene.queries.function.docvalues.LongDocValues; +import org.apache.lucene.queries.function.valuesource.ComparisonBoolFunction; + +/** + * Refines {@link ComparisonBoolFunction} to compare based on a 'long' or 'double' depending on if the + * any of the FunctionValues are {@link LongDocValues}. + */ +public class SolrComparisonBoolFunction extends ComparisonBoolFunction { + + private final Compare cmp; + + public interface Compare { + boolean compare(int integer); + } + + public SolrComparisonBoolFunction(ValueSource lhs, ValueSource rhs, String name, Compare cmp) { + super(lhs, rhs, name); + this.cmp = cmp; + } + + @Override + public boolean compare(int doc, FunctionValues lhs, FunctionValues rhs) { + // TODO consider a separate FunctionValues impl, one for Long, one for Double + // performs the safest possible numeric comparison, if both lhs and rhs are Longs, then + // we perform a Long comparison to avoid the issues with precision when casting to doubles + boolean lhsAnInt = (lhs instanceof LongDocValues || lhs instanceof IntDocValues); + boolean rhsAnInt = (rhs instanceof LongDocValues || rhs instanceof IntDocValues); + if (lhsAnInt && rhsAnInt) { + return cmp.compare(Long.compare(lhs.longVal(doc), rhs.longVal(doc))); + } else { + return cmp.compare(Double.compare(lhs.doubleVal(doc), rhs.doubleVal(doc))); + } + } + + // note: don't override equals; the "name" will be unique and is already compared +} diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java index a30ff8387ba..86fe729447e 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/command/QueryCommand.java @@ -62,7 +62,7 @@ public class QueryCommand implements Command { * @return this */ public Builder setQuery(String groupQueryString, SolrQueryRequest request) throws SyntaxError { - QParser parser = QParser.getParser(groupQueryString, null, request); + QParser parser = QParser.getParser(groupQueryString, request); this.queryString = groupQueryString; return setQuery(parser.getQuery()); } diff --git a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java index cced17ece9e..999cd648ad7 100644 --- a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java @@ -236,7 +236,7 @@ public class ScoreJoinQParserPlugin extends QParserPlugin { LocalSolrQueryRequest otherReq = new LocalSolrQueryRequest(fromCore, params); try { - QParser fromQueryParser = QParser.getParser(fromQueryStr, "lucene", otherReq); + QParser fromQueryParser = QParser.getParser(fromQueryStr, otherReq); Query fromQuery = fromQueryParser.getQuery(); fromHolder = fromCore.getRegisteredSearcher(); diff --git a/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java b/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java index 105f307c055..d8f2ef21279 100644 --- a/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java @@ -17,18 +17,11 @@ package org.apache.solr.security; import javax.servlet.FilterChain; -import javax.servlet.ServletException; import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletRequestWrapper; import java.io.Closeable; -import java.io.IOException; -import java.security.Principal; import java.util.Map; -import org.apache.http.auth.BasicUserPrincipal; - /** * * @lucene.experimental @@ -42,32 +35,20 @@ public abstract class AuthenticationPlugin implements Closeable { * @param pluginConfig Config parameters, possibly from a ZK source */ public abstract void init(Map pluginConfig); - - protected void forward(String user, ServletRequest req, ServletResponse rsp, - FilterChain chain) throws IOException, ServletException { - if(user != null) { - final Principal p = new BasicUserPrincipal(user); - req = new HttpServletRequestWrapper((HttpServletRequest) req) { - @Override - public Principal getUserPrincipal() { - return p; - } - }; - } - chain.doFilter(req,rsp); - } /** - * This method must authenticate the request. Upon a successful authentication, this + * This method attempts to authenticate the request. Upon a successful authentication, this * must call the next filter in the filter chain and set the user principal of the request, * or else, upon an error or an authentication failure, throw an exception. - * + * * @param request the http request * @param response the http response * @param filterChain the servlet filter chain + * @return false if the request not be processed by Solr (not continue), i.e. + * the response and status code have already been sent. * @throws Exception any exception thrown during the authentication, e.g. PrivilegedActionException */ - public abstract void doAuthenticate(ServletRequest request, ServletResponse response, + public abstract boolean doAuthenticate(ServletRequest request, ServletResponse response, FilterChain filterChain) throws Exception; diff --git a/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java index 03c75c631a2..e3f53a26ae8 100644 --- a/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java @@ -99,7 +99,7 @@ public class BasicAuthPlugin extends AuthenticationPlugin implements ConfigEdita } @Override - public void doAuthenticate(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws Exception { + public boolean doAuthenticate(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws Exception { HttpServletRequest request = (HttpServletRequest) servletRequest; HttpServletResponse response = (HttpServletResponse) servletResponse; @@ -127,6 +127,7 @@ public class BasicAuthPlugin extends AuthenticationPlugin implements ConfigEdita } }; filterChain.doFilter(wrapper, response); + return true; } } else { @@ -143,8 +144,10 @@ public class BasicAuthPlugin extends AuthenticationPlugin implements ConfigEdita } else { request.setAttribute(AuthenticationPlugin.class.getName(), zkAuthentication.getPromptHeaders()); filterChain.doFilter(request, response); + return true; } } + return false; } @Override diff --git a/solr/core/src/java/org/apache/solr/security/DelegationTokenKerberosFilter.java b/solr/core/src/java/org/apache/solr/security/DelegationTokenKerberosFilter.java new file mode 100644 index 00000000000..ca27861e8c4 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/DelegationTokenKerberosFilter.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Enumeration; +import java.util.LinkedList; +import java.util.List; + +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; + +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.AuthInfo; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.api.ACLProvider; +import org.apache.curator.retry.ExponentialBackoffRetry; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.server.AuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; +import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation; +import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkACLProvider; +import org.apache.solr.common.cloud.ZkCredentialsProvider; +import org.apache.zookeeper.data.ACL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DelegationTokenKerberosFilter extends DelegationTokenAuthenticationFilter { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private CuratorFramework curatorFramework; + + @Override + public void init(FilterConfig conf) throws ServletException { + if (conf != null && "zookeeper".equals(conf.getInitParameter("signer.secret.provider"))) { + SolrZkClient zkClient = + (SolrZkClient)conf.getServletContext().getAttribute(KerberosPlugin.DELEGATION_TOKEN_ZK_CLIENT); + conf.getServletContext().setAttribute("signer.secret.provider.zookeeper.curator.client", + getCuratorClient(zkClient)); + } + super.init(conf); + } + + /** + * Return the ProxyUser Configuration. FilterConfig properties beginning with + * "solr.impersonator.user.name" will be added to the configuration. + */ + @Override + protected Configuration getProxyuserConfiguration(FilterConfig filterConf) + throws ServletException { + Configuration conf = new Configuration(false); + + Enumeration names = filterConf.getInitParameterNames(); + while (names.hasMoreElements()) { + String name = (String) names.nextElement(); + if (name.startsWith(KerberosPlugin.IMPERSONATOR_PREFIX)) { + String value = filterConf.getInitParameter(name); + conf.set(PROXYUSER_PREFIX + "." + name.substring(KerberosPlugin.IMPERSONATOR_PREFIX.length()), value); + conf.set(name, value); + } + } + return conf; + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain filterChain) throws IOException, ServletException { + // HttpClient 4.4.x throws NPE if query string is null and parsed through URLEncodedUtils. + // See HTTPCLIENT-1746 and HADOOP-12767 + HttpServletRequest httpRequest = (HttpServletRequest)request; + String queryString = httpRequest.getQueryString(); + final String nonNullQueryString = queryString == null ? "" : queryString; + HttpServletRequest requestNonNullQueryString = new HttpServletRequestWrapper(httpRequest){ + @Override + public String getQueryString() { + return nonNullQueryString; + } + }; + + // include Impersonator User Name in case someone (e.g. logger) wants it + FilterChain filterChainWrapper = new FilterChain() { + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse) + throws IOException, ServletException { + HttpServletRequest httpRequest = (HttpServletRequest) servletRequest; + + UserGroupInformation ugi = HttpUserGroupInformation.get(); + if (ugi != null && ugi.getAuthenticationMethod() == UserGroupInformation.AuthenticationMethod.PROXY) { + UserGroupInformation realUserUgi = ugi.getRealUser(); + if (realUserUgi != null) { + httpRequest.setAttribute(KerberosPlugin.IMPERSONATOR_USER_NAME, realUserUgi.getShortUserName()); + } + } + filterChain.doFilter(servletRequest, servletResponse); + } + }; + + super.doFilter(requestNonNullQueryString, response, filterChainWrapper); + } + + @Override + public void destroy() { + super.destroy(); + if (curatorFramework != null) curatorFramework.close(); + curatorFramework = null; + } + + @Override + protected void initializeAuthHandler(String authHandlerClassName, + FilterConfig filterConfig) throws ServletException { + // set the internal authentication handler in order to record whether the request should continue + super.initializeAuthHandler(authHandlerClassName, filterConfig); + AuthenticationHandler authHandler = getAuthenticationHandler(); + super.initializeAuthHandler(KerberosPlugin.RequestContinuesRecorderAuthenticationHandler.class.getName(), filterConfig); + KerberosPlugin.RequestContinuesRecorderAuthenticationHandler newAuthHandler = + (KerberosPlugin.RequestContinuesRecorderAuthenticationHandler)getAuthenticationHandler(); + newAuthHandler.setAuthHandler(authHandler); + } + + protected CuratorFramework getCuratorClient(SolrZkClient zkClient) { + // should we try to build a RetryPolicy off of the ZkController? + RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); + if (zkClient == null) { + throw new IllegalArgumentException("zkClient required"); + } + String zkHost = zkClient.getZkServerAddress(); + String zkChroot = zkHost.substring(zkHost.indexOf("/")); + zkChroot = zkChroot.startsWith("/") ? zkChroot.substring(1) : zkChroot; + String zkNamespace = zkChroot + SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH; + String zkConnectionString = zkHost.substring(0, zkHost.indexOf("/")); + SolrZkToCuratorCredentialsACLs curatorToSolrZk = new SolrZkToCuratorCredentialsACLs(zkClient); + final int connectionTimeoutMs = 30000; // this value is currently hard coded, see SOLR-7561. + + curatorFramework = CuratorFrameworkFactory.builder() + .namespace(zkNamespace) + .connectString(zkConnectionString) + .retryPolicy(retryPolicy) + .aclProvider(curatorToSolrZk.getACLProvider()) + .authorization(curatorToSolrZk.getAuthInfos()) + .sessionTimeoutMs(zkClient.getZkClientTimeout()) + .connectionTimeoutMs(connectionTimeoutMs) + .build(); + curatorFramework.start(); + return curatorFramework; + } + + /** + * Convert Solr Zk Credentials/ACLs to Curator versions + */ + protected static class SolrZkToCuratorCredentialsACLs { + private final ACLProvider aclProvider; + private final List authInfos; + + public SolrZkToCuratorCredentialsACLs(SolrZkClient zkClient) { + this.aclProvider = createACLProvider(zkClient); + this.authInfos = createAuthInfo(zkClient); + } + + public ACLProvider getACLProvider() { return aclProvider; } + public List getAuthInfos() { return authInfos; } + + private ACLProvider createACLProvider(SolrZkClient zkClient) { + final ZkACLProvider zkACLProvider = zkClient.getZkACLProvider(); + return new ACLProvider() { + @Override + public List getDefaultAcl() { + return zkACLProvider.getACLsToAdd(null); + } + + @Override + public List getAclForPath(String path) { + List acls = zkACLProvider.getACLsToAdd(path); + return acls; + } + }; + } + + private List createAuthInfo(SolrZkClient zkClient) { + List ret = new LinkedList(); + + // In theory the credentials to add could change here if zookeeper hasn't been initialized + ZkCredentialsProvider credentialsProvider = + zkClient.getZkClientConnectionStrategy().getZkCredentialsToAddAutomatically(); + for (ZkCredentialsProvider.ZkCredentials zkCredentials : credentialsProvider.getCredentials()) { + ret.add(new AuthInfo(zkCredentials.getScheme(), zkCredentials.getAuth())); + } + return ret; + } + } +} diff --git a/solr/core/src/java/org/apache/solr/security/KerberosFilter.java b/solr/core/src/java/org/apache/solr/security/KerberosFilter.java index ee234882491..9c5305056d2 100644 --- a/solr/core/src/java/org/apache/solr/security/KerberosFilter.java +++ b/solr/core/src/java/org/apache/solr/security/KerberosFilter.java @@ -26,6 +26,7 @@ import javax.servlet.ServletResponse; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.security.authentication.server.AuthenticationHandler; public class KerberosFilter extends AuthenticationFilter { @@ -34,6 +35,19 @@ public class KerberosFilter extends AuthenticationFilter { super.init(conf); } + @Override + protected void initializeAuthHandler(String authHandlerClassName, + FilterConfig filterConfig) throws ServletException { + // set the internal authentication handler in order to record whether the request should continue + super.initializeAuthHandler(authHandlerClassName, filterConfig); + AuthenticationHandler authHandler = getAuthenticationHandler(); + super.initializeAuthHandler( + KerberosPlugin.RequestContinuesRecorderAuthenticationHandler.class.getName(), filterConfig); + KerberosPlugin.RequestContinuesRecorderAuthenticationHandler newAuthHandler = + (KerberosPlugin.RequestContinuesRecorderAuthenticationHandler)getAuthenticationHandler(); + newAuthHandler.setAuthHandler(authHandler); + } + @Override protected void doFilter(FilterChain filterChain, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { diff --git a/solr/core/src/java/org/apache/solr/security/KerberosPlugin.java b/solr/core/src/java/org/apache/solr/security/KerberosPlugin.java index 7a83ab527ab..d4a282359ef 100644 --- a/solr/core/src/java/org/apache/solr/security/KerberosPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/KerberosPlugin.java @@ -16,14 +16,18 @@ */ package org.apache.solr.security; +import java.io.IOException; import java.io.InputStream; +import java.io.PrintWriter; import java.lang.invoke.MethodHandles; import java.net.MalformedURLException; import java.net.URL; +import java.util.Collections; import java.util.Enumeration; import java.util.EventListener; import java.util.HashMap; import java.util.Map; +import java.util.Properties; import java.util.Set; import javax.servlet.Filter; @@ -41,12 +45,22 @@ import javax.servlet.ServletResponse; import javax.servlet.SessionCookieConfig; import javax.servlet.SessionTrackingMode; import javax.servlet.descriptor.JspConfigDescriptor; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletResponseWrapper; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.collections.iterators.IteratorEnumeration; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.server.AuthenticationHandler; +import org.apache.hadoop.security.authentication.server.AuthenticationToken; import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder; import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; +import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; +import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.core.CoreContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,7 +69,7 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); Krb5HttpClientBuilder kerberosBuilder = new Krb5HttpClientBuilder(); - Filter kerberosFilter = new KerberosFilter(); + private Filter kerberosFilter; public static final String NAME_RULES_PARAM = "solr.kerberos.name.rules"; public static final String COOKIE_DOMAIN_PARAM = "solr.kerberos.cookie.domain"; @@ -64,6 +78,26 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu public static final String KEYTAB_PARAM = "solr.kerberos.keytab"; public static final String TOKEN_VALID_PARAM = "solr.kerberos.token.valid"; public static final String COOKIE_PORT_AWARE_PARAM = "solr.kerberos.cookie.portaware"; + public static final String IMPERSONATOR_PREFIX = "solr.kerberos.impersonator.user."; + public static final String DELEGATION_TOKEN_ENABLED = "solr.kerberos.delegation.token.enabled"; + public static final String DELEGATION_TOKEN_KIND = "solr.kerberos.delegation.token.kind"; + public static final String DELEGATION_TOKEN_VALIDITY = "solr.kerberos.delegation.token.validity"; + public static final String DELEGATION_TOKEN_SECRET_PROVIDER = "solr.kerberos.delegation.token.signer.secret.provider"; + public static final String DELEGATION_TOKEN_SECRET_PROVIDER_ZK_PATH = + "solr.kerberos.delegation.token.signer.secret.provider.zookeper.path"; + public static final String DELEGATION_TOKEN_SECRET_MANAGER_ZNODE_WORKING_PATH = + "solr.kerberos.delegation.token.secret.manager.znode.working.path"; + + public static final String DELEGATION_TOKEN_TYPE_DEFAULT = "solr-dt"; + public static final String IMPERSONATOR_DO_AS_HTTP_PARAM = "doAs"; + public static final String IMPERSONATOR_USER_NAME = "solr.impersonator.user.name"; + + // filled in by Plugin/Filter + static final String REQUEST_CONTINUES_ATTR = + "org.apache.solr.security.kerberosplugin.requestcontinues"; + static final String DELEGATION_TOKEN_ZK_CLIENT = + "solr.kerberos.delegation.token.zk.client"; + private final CoreContainer coreContainer; public KerberosPlugin(CoreContainer coreContainer) { @@ -73,61 +107,123 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu @Override public void init(Map pluginConfig) { try { - Map params = new HashMap(); - params.put("type", "kerberos"); - putParam(params, "kerberos.name.rules", NAME_RULES_PARAM, "DEFAULT"); - putParam(params, "token.valid", TOKEN_VALID_PARAM, "30"); - putParam(params, "cookie.path", COOKIE_PATH_PARAM, "/"); - putParam(params, "kerberos.principal", PRINCIPAL_PARAM, null); - putParam(params, "kerberos.keytab", KEYTAB_PARAM, null); - - // Special handling for the "cookie.domain" based on whether port should be - // appended to the domain. Useful for situations where multiple solr nodes are - // on the same host. - String usePortStr = System.getProperty(COOKIE_PORT_AWARE_PARAM, null); - boolean needPortAwareCookies = (usePortStr == null) ? false: Boolean.parseBoolean(usePortStr); - - if (!needPortAwareCookies || !coreContainer.isZooKeeperAware()) { - putParam(params, "cookie.domain", COOKIE_DOMAIN_PARAM, null); - } else { // we need port aware cookies and we are in SolrCloud mode. - String host = System.getProperty(COOKIE_DOMAIN_PARAM, null); - if (host==null) { - throw new SolrException(ErrorCode.SERVER_ERROR, "Missing required parameter '"+COOKIE_DOMAIN_PARAM+"'."); - } - int port = coreContainer.getZkController().getHostPort(); - params.put("cookie.domain", host + ":" + port); - } - - log.info("Params: "+params); - - FilterConfig conf = new FilterConfig() { - @Override - public ServletContext getServletContext() { - return noContext; - } - - @Override - public Enumeration getInitParameterNames() { - return new IteratorEnumeration(params.keySet().iterator()); - } - - @Override - public String getInitParameter(String param) { - return params.get(param); - } - - @Override - public String getFilterName() { - return "KerberosFilter"; - } - }; - + FilterConfig conf = getInitFilterConfig(pluginConfig, false); kerberosFilter.init(conf); } catch (ServletException e) { throw new SolrException(ErrorCode.SERVER_ERROR, "Error initializing kerberos authentication plugin: "+e); } } + @VisibleForTesting + protected FilterConfig getInitFilterConfig(Map pluginConfig, boolean skipKerberosChecking) { + Map params = new HashMap(); + params.put("type", "kerberos"); + putParam(params, "kerberos.name.rules", NAME_RULES_PARAM, "DEFAULT"); + putParam(params, "token.valid", TOKEN_VALID_PARAM, "30"); + putParam(params, "cookie.path", COOKIE_PATH_PARAM, "/"); + if (!skipKerberosChecking) { + putParam(params, "kerberos.principal", PRINCIPAL_PARAM, null); + putParam(params, "kerberos.keytab", KEYTAB_PARAM, null); + } else { + putParamOptional(params, "kerberos.principal", PRINCIPAL_PARAM); + putParamOptional(params, "kerberos.keytab", KEYTAB_PARAM); + } + + String delegationTokenStr = System.getProperty(DELEGATION_TOKEN_ENABLED, null); + boolean delegationTokenEnabled = + (delegationTokenStr == null) ? false : Boolean.parseBoolean(delegationTokenStr); + ZkController controller = coreContainer.getZkController(); + + if (delegationTokenEnabled) { + putParam(params, "delegation-token.token-kind", DELEGATION_TOKEN_KIND, DELEGATION_TOKEN_TYPE_DEFAULT); + if (coreContainer.isZooKeeperAware()) { + putParam(params, "signer.secret.provider", DELEGATION_TOKEN_SECRET_PROVIDER, "zookeeper"); + if ("zookeeper".equals(params.get("signer.secret.provider"))) { + String zkHost = controller.getZkServerAddress(); + putParam(params, "token.validity", DELEGATION_TOKEN_VALIDITY, "36000"); + params.put("zk-dt-secret-manager.enable", "true"); + // Note - Curator complains if the znodeWorkingPath starts with / + String chrootPath = zkHost.substring(zkHost.indexOf("/")); + String relativePath = chrootPath.startsWith("/") ? chrootPath.substring(1) : chrootPath; + putParam(params, "zk-dt-secret-manager.znodeWorkingPath", + DELEGATION_TOKEN_SECRET_MANAGER_ZNODE_WORKING_PATH, + relativePath + SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH + "/zkdtsm"); + putParam(params, "signer.secret.provider.zookeeper.path", + DELEGATION_TOKEN_SECRET_PROVIDER_ZK_PATH, "/token"); + // ensure krb5 is setup properly before running curator + getHttpClientBuilder(SolrHttpClientBuilder.create()); + } + } else { + log.info("CoreContainer is not ZooKeeperAware, not setting ZK-related delegation token properties"); + } + } + + // Special handling for the "cookie.domain" based on whether port should be + // appended to the domain. Useful for situations where multiple solr nodes are + // on the same host. + String usePortStr = System.getProperty(COOKIE_PORT_AWARE_PARAM, null); + boolean needPortAwareCookies = (usePortStr == null) ? false: Boolean.parseBoolean(usePortStr); + + if (!needPortAwareCookies || !coreContainer.isZooKeeperAware()) { + putParam(params, "cookie.domain", COOKIE_DOMAIN_PARAM, null); + } else { // we need port aware cookies and we are in SolrCloud mode. + String host = System.getProperty(COOKIE_DOMAIN_PARAM, null); + if (host==null) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Missing required parameter '"+COOKIE_DOMAIN_PARAM+"'."); + } + int port = controller.getHostPort(); + params.put("cookie.domain", host + ":" + port); + } + + // check impersonator config + for (Enumeration e = System.getProperties().propertyNames(); e.hasMoreElements();) { + String key = e.nextElement().toString(); + if (key.startsWith(IMPERSONATOR_PREFIX)) { + if (!delegationTokenEnabled) { + throw new SolrException(ErrorCode.SERVER_ERROR, + "Impersonator configuration requires delegation tokens to be enabled: " + key); + } + params.put(key, System.getProperty(key)); + } + } + final ServletContext servletContext = new AttributeOnlyServletContext(); + if (controller != null) { + servletContext.setAttribute(DELEGATION_TOKEN_ZK_CLIENT, controller.getZkClient()); + } + if (delegationTokenEnabled) { + kerberosFilter = new DelegationTokenKerberosFilter(); + // pass an attribute-enabled context in order to pass the zkClient + // and because the filter may pass a curator instance. + } else { + kerberosFilter = new KerberosFilter(); + } + log.info("Params: "+params); + + FilterConfig conf = new FilterConfig() { + @Override + public ServletContext getServletContext() { + return servletContext; + } + + @Override + public Enumeration getInitParameterNames() { + return new IteratorEnumeration(params.keySet().iterator()); + } + + @Override + public String getInitParameter(String param) { + return params.get(param); + } + + @Override + public String getFilterName() { + return "KerberosFilter"; + } + }; + + return conf; + } + private void putParam(Map params, String internalParamName, String externalParamName, String defaultValue) { String value = System.getProperty(externalParamName, defaultValue); if (value==null) { @@ -136,11 +232,43 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu params.put(internalParamName, value); } + private void putParamOptional(Map params, String internalParamName, String externalParamName) { + String value = System.getProperty(externalParamName); + if (value!=null) { + params.put(internalParamName, value); + } + } + @Override - public void doAuthenticate(ServletRequest req, ServletResponse rsp, + public boolean doAuthenticate(ServletRequest req, ServletResponse rsp, FilterChain chain) throws Exception { log.debug("Request to authenticate using kerberos: "+req); - kerberosFilter.doFilter(req, rsp, chain); + + final HttpServletResponse frsp = (HttpServletResponse)rsp; + + // kerberosFilter may close the stream and write to closed streams, + // see HADOOP-13346. To work around, pass a PrintWriter that ignores + // closes + HttpServletResponse rspCloseShield = new HttpServletResponseWrapper(frsp) { + @SuppressForbidden(reason = "Hadoop DelegationTokenAuthenticationFilter uses response writer, this" + + "is providing a CloseShield on top of that") + @Override + public PrintWriter getWriter() throws IOException { + final PrintWriter pw = new PrintWriterWrapper(frsp.getWriter()) { + @Override + public void close() {}; + }; + return pw; + } + }; + kerberosFilter.doFilter(req, rspCloseShield, chain); + String requestContinuesAttr = (String)req.getAttribute(REQUEST_CONTINUES_ATTR); + if (requestContinuesAttr == null) { + log.warn("Could not find " + REQUEST_CONTINUES_ATTR); + return false; + } else { + return Boolean.parseBoolean(requestContinuesAttr); + } } @Override @@ -148,13 +276,19 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu return kerberosBuilder.getBuilder(builder); } + @Override public void close() { kerberosFilter.destroy(); kerberosBuilder.close(); } - protected static ServletContext noContext = new ServletContext() { - + protected Filter getKerberosFilter() { return kerberosFilter; } + + protected void setKerberosFilter(Filter kerberosFilter) { this.kerberosFilter = kerberosFilter; } + + protected static class AttributeOnlyServletContext implements ServletContext { + private Map attributes = new HashMap(); + @Override public void setSessionTrackingModes(Set sessionTrackingModes) {} @@ -162,12 +296,16 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu public boolean setInitParameter(String name, String value) { return false; } - + @Override - public void setAttribute(String name, Object object) {} - + public void setAttribute(String name, Object object) { + attributes.put(name, object); + } + @Override - public void removeAttribute(String name) {} + public void removeAttribute(String name) { + attributes.remove(name); + } @Override public void log(String message, Throwable throwable) {} @@ -327,15 +465,15 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu public ClassLoader getClassLoader() { return null; } - + @Override public Enumeration getAttributeNames() { - return null; + return Collections.enumeration(attributes.keySet()); } - + @Override public Object getAttribute(String name) { - return null; + return attributes.get(name); } @Override @@ -395,4 +533,44 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu return null; } }; + + /* + * {@link AuthenticationHandler} that delegates to another {@link AuthenticationHandler} + * and records the response of managementOperation (which indicates whether the request + * should continue or not). + */ + public static class RequestContinuesRecorderAuthenticationHandler implements AuthenticationHandler { + private AuthenticationHandler authHandler; + + public void setAuthHandler(AuthenticationHandler authHandler) { + this.authHandler = authHandler; + } + + public String getType() { + return authHandler.getType(); + } + + public void init(Properties config) throws ServletException { + // authHandler has already been init'ed, nothing to do here + } + + public void destroy() { + authHandler.destroy(); + } + + public boolean managementOperation(AuthenticationToken token, + HttpServletRequest request, + HttpServletResponse response) + throws IOException, AuthenticationException { + boolean result = authHandler.managementOperation(token, request, response); + request.setAttribute(KerberosPlugin.REQUEST_CONTINUES_ATTR, new Boolean(result).toString()); + return result; + } + + + public AuthenticationToken authenticate(HttpServletRequest request, HttpServletResponse response) + throws IOException, AuthenticationException { + return authHandler.authenticate(request, response); + } + } } diff --git a/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java b/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java index fb3665c0f92..e5d26531fbe 100644 --- a/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java @@ -89,12 +89,12 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt @SuppressForbidden(reason = "Needs currentTimeMillis to compare against time in header") @Override - public void doAuthenticate(ServletRequest request, ServletResponse response, FilterChain filterChain) throws Exception { + public boolean doAuthenticate(ServletRequest request, ServletResponse response, FilterChain filterChain) throws Exception { String requestURI = ((HttpServletRequest) request).getRequestURI(); if (requestURI.endsWith(PATH)) { filterChain.doFilter(request, response); - return; + return true; } long receivedTime = System.currentTimeMillis(); String header = ((HttpServletRequest) request).getHeader(HEADER); @@ -102,14 +102,14 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt //this must not happen log.error("No SolrAuth header present"); filterChain.doFilter(request, response); - return; + return true; } List authInfo = StrUtils.splitWS(header, false); if (authInfo.size() < 2) { log.error("Invalid SolrAuth Header {}", header); filterChain.doFilter(request, response); - return; + return true; } String nodeName = authInfo.get(0); @@ -119,12 +119,12 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt if (decipher == null) { log.error("Could not decipher a header {} . No principal set", header); filterChain.doFilter(request, response); - return; + return true; } if ((receivedTime - decipher.timestamp) > MAX_VALIDITY) { log.error("Invalid key request timestamp: {} , received timestamp: {} , TTL: {}", decipher.timestamp, receivedTime, MAX_VALIDITY); filterChain.doFilter(request, response); - return; + return true; } final Principal principal = "$".equals(decipher.userName) ? @@ -132,6 +132,7 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt new BasicUserPrincipal(decipher.userName); filterChain.doFilter(getWrapper((HttpServletRequest) request, principal), response); + return true; } private static HttpServletRequestWrapper getWrapper(final HttpServletRequest request, final Principal principal) { diff --git a/solr/core/src/java/org/apache/solr/security/PrintWriterWrapper.java b/solr/core/src/java/org/apache/solr/security/PrintWriterWrapper.java new file mode 100644 index 00000000000..a4e47b5e4f1 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/security/PrintWriterWrapper.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Locale; + +import org.apache.commons.lang.NotImplementedException; + +/** + * Wrapper for PrintWriter that delegates to constructor arg + */ +public class PrintWriterWrapper extends PrintWriter { + private PrintWriter printWriter; + + public PrintWriterWrapper(PrintWriter printWriter) { + super(new StringWriter()); + this.printWriter = printWriter; + } + + @Override + public PrintWriter append(char c) { + return printWriter.append(c); + } + + @Override + public PrintWriter append(CharSequence csq) { + return printWriter.append(csq); + } + + @Override + public PrintWriter append(CharSequence csq, int start, int end) { + return printWriter.append(csq, start, end); + } + + @Override + public boolean checkError() { + return printWriter.checkError(); + } + + @Override + protected void clearError() { + throw new NotImplementedException(); + } + + @Override + public void close() { + printWriter.close(); + } + + @Override + public void flush() { + printWriter.flush(); + } + + @Override + public PrintWriter format(Locale l, String format, Object... args) { + return printWriter.format(l, format, args); + } + + @Override + public PrintWriter format(String format, Object... args) { + throw new NotImplementedException("Forbidden API"); + } + + @Override + public void print(boolean b) { + printWriter.print(b); + } + + @Override + public void print(char c) { + printWriter.print(c); + } + + @Override + public void print(char[] s) { + printWriter.print(s); + } + + @Override + public void print(double d) { + printWriter.print(d); + } + + @Override + public void print(float f) { + printWriter.print(f); + } + + @Override + public void print(int i) { + printWriter.print(i); + } + + @Override + public void print(long l) { + printWriter.print(l); + } + + @Override + public void print(Object obj) { + printWriter.print(obj); + } + + @Override + public void print(String s) { + printWriter.print(s); + } + + @Override + public PrintWriter printf(Locale l, String format, Object... args) { + return printWriter.printf(l, format, args); + } + + @Override + public PrintWriter printf(String format, Object... args) { + throw new NotImplementedException("Forbidden API"); + } + + @Override + public void println() { + printWriter.println(); + } + + @Override + public void println(boolean x) { + printWriter.println(x); + } + + @Override + public void println(char x) { + printWriter.println(x); + } + + @Override + public void println(char[] x) { + printWriter.println(x); + } + + @Override + public void println(double x) { + printWriter.println(x); + } + + @Override + public void println(float x) { + printWriter.println(x); + } + + @Override + public void println(int x) { + printWriter.println(x); + } + + @Override + public void println(long x) { + printWriter.println(x); + } + + @Override + public void println(Object x) { + printWriter.println(x); + } + + @Override + public void println(String x) { + printWriter.println(x); + } + + @Override + protected void setError() { + throw new NotImplementedException(); + } + + @Override + public void write(char[] buf) { + printWriter.write(buf); + } + + @Override + public void write(char[] buf, int off, int len) { + printWriter.write(buf, off, len); + } + + @Override + public void write(int c) { + printWriter.write(c); + } + + @Override + public void write(String s) { + printWriter.write(s); + } + + @Override + public void write(String s, int off, int len) { + printWriter.write(s, off, len); + } +} diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 2d089350ddc..4a680e5591c 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -296,6 +296,7 @@ public class SolrDispatchFilter extends BaseSolrFilter { } private boolean authenticateRequest(ServletRequest request, ServletResponse response, final AtomicReference wrappedRequest) throws IOException { + boolean requestContinues = false; final AtomicBoolean isAuthenticated = new AtomicBoolean(false); AuthenticationPlugin authenticationPlugin = cores.getAuthenticationPlugin(); if (authenticationPlugin == null) { @@ -308,7 +309,7 @@ public class SolrDispatchFilter extends BaseSolrFilter { try { log.debug("Request to authenticate: {}, domain: {}, port: {}", request, request.getLocalName(), request.getLocalPort()); // upon successful authentication, this should call the chain's next filter. - authenticationPlugin.doAuthenticate(request, response, new FilterChain() { + requestContinues = authenticationPlugin.doAuthenticate(request, response, new FilterChain() { public void doFilter(ServletRequest req, ServletResponse rsp) throws IOException, ServletException { isAuthenticated.set(true); wrappedRequest.set(req); @@ -319,8 +320,13 @@ public class SolrDispatchFilter extends BaseSolrFilter { throw new SolrException(ErrorCode.SERVER_ERROR, "Error during request authentication, ", e); } } - // failed authentication? - if (!isAuthenticated.get()) { + // requestContinues is an optional short circuit, thus we still need to check isAuthenticated. + // This is because the AuthenticationPlugin doesn't always have enough information to determine if + // it should short circuit, e.g. the Kerberos Authentication Filter will send an error and not + // call later filters in chain, but doesn't throw an exception. We could force each Plugin + // to implement isAuthenticated to simplify the check here, but that just moves the complexity to + // multiple code paths. + if (!requestContinues || !isAuthenticated.get()) { response.flushBuffer(); return false; } diff --git a/solr/core/src/java/org/apache/solr/update/CdcrUpdateLog.java b/solr/core/src/java/org/apache/solr/update/CdcrUpdateLog.java index 3afc66c24d8..6b202044d76 100644 --- a/solr/core/src/java/org/apache/solr/update/CdcrUpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/CdcrUpdateLog.java @@ -151,7 +151,12 @@ public class CdcrUpdateLog extends UpdateLog { if (id != -1) return id; if (tlogFiles.length == 0) return -1; String last = tlogFiles[tlogFiles.length - 1]; - return Long.parseLong(last.substring(TLOG_NAME.length() + 1, last.lastIndexOf('.'))); + if (TLOG_NAME.length() + 1 > last.lastIndexOf('.')) { + // old tlog created by default UpdateLog impl + return Long.parseLong(last.substring(TLOG_NAME.length() + 1)); + } else { + return Long.parseLong(last.substring(TLOG_NAME.length() + 1, last.lastIndexOf('.'))); + } } @Override diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java index a29d57dcb5f..c57ee750d86 100644 --- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java @@ -395,5 +395,9 @@ public final class DefaultSolrCoreState extends SolrCoreState implements Recover public void setLastReplicateIndexSuccess(boolean success) { this.lastReplicationSuccess = success; } - + + @Override + public Lock getRecoveryLock() { + return recoveryLock; + } } diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java index 8c3c74964c2..0bdefa7c9d8 100644 --- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -26,7 +26,7 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; import org.apache.lucene.document.Document; import org.apache.lucene.index.CodecReader; @@ -76,20 +76,20 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState protected final SolrCoreState solrCoreState; // stats - AtomicLong addCommands = new AtomicLong(); - AtomicLong addCommandsCumulative = new AtomicLong(); - AtomicLong deleteByIdCommands= new AtomicLong(); - AtomicLong deleteByIdCommandsCumulative= new AtomicLong(); - AtomicLong deleteByQueryCommands= new AtomicLong(); - AtomicLong deleteByQueryCommandsCumulative= new AtomicLong(); - AtomicLong expungeDeleteCommands = new AtomicLong(); - AtomicLong mergeIndexesCommands = new AtomicLong(); - AtomicLong commitCommands= new AtomicLong(); - AtomicLong optimizeCommands= new AtomicLong(); - AtomicLong rollbackCommands= new AtomicLong(); - AtomicLong numDocsPending= new AtomicLong(); - AtomicLong numErrors = new AtomicLong(); - AtomicLong numErrorsCumulative = new AtomicLong(); + LongAdder addCommands = new LongAdder(); + LongAdder addCommandsCumulative = new LongAdder(); + LongAdder deleteByIdCommands= new LongAdder(); + LongAdder deleteByIdCommandsCumulative= new LongAdder(); + LongAdder deleteByQueryCommands= new LongAdder(); + LongAdder deleteByQueryCommandsCumulative= new LongAdder(); + LongAdder expungeDeleteCommands = new LongAdder(); + LongAdder mergeIndexesCommands = new LongAdder(); + LongAdder commitCommands= new LongAdder(); + LongAdder optimizeCommands= new LongAdder(); + LongAdder rollbackCommands= new LongAdder(); + LongAdder numDocsPending= new LongAdder(); + LongAdder numErrors = new LongAdder(); + LongAdder numErrorsCumulative = new LongAdder(); // tracks when auto-commit should occur protected final CommitTracker commitTracker; @@ -158,7 +158,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState } protected void rollbackWriter() throws IOException { - numDocsPending.set(0); + numDocsPending.reset(); solrCoreState.rollbackIndexWriter(core); } @@ -192,8 +192,8 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState private int addDoc0(AddUpdateCommand cmd) throws IOException { int rc = -1; - addCommands.incrementAndGet(); - addCommandsCumulative.incrementAndGet(); + addCommands.increment(); + addCommandsCumulative.increment(); // if there is no ID field, don't overwrite if (idField == null) { @@ -230,10 +230,10 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState rc = 1; } finally { if (rc != 1) { - numErrors.incrementAndGet(); - numErrorsCumulative.incrementAndGet(); + numErrors.increment(); + numErrorsCumulative.increment(); } else { - numDocsPending.incrementAndGet(); + numDocsPending.increment(); } } @@ -368,8 +368,8 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState // we don't return the number of docs deleted because it's not always possible to quickly know that info. @Override public void delete(DeleteUpdateCommand cmd) throws IOException { - deleteByIdCommands.incrementAndGet(); - deleteByIdCommandsCumulative.incrementAndGet(); + deleteByIdCommands.increment(); + deleteByIdCommandsCumulative.increment(); Term deleteTerm = new Term(idField.getName(), cmd.getIndexedId()); // SolrCore.verbose("deleteDocuments",deleteTerm,writer); @@ -399,7 +399,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState Query q; try { // move this higher in the stack? - QParser parser = QParser.getParser(cmd.getQuery(), "lucene", cmd.req); + QParser parser = QParser.getParser(cmd.getQuery(), cmd.req); q = parser.getQuery(); q = QueryUtils.makeQueryable(q); @@ -426,8 +426,8 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState // we don't return the number of docs deleted because it's not always possible to quickly know that info. @Override public void deleteByQuery(DeleteUpdateCommand cmd) throws IOException { - deleteByQueryCommands.incrementAndGet(); - deleteByQueryCommandsCumulative.incrementAndGet(); + deleteByQueryCommands.increment(); + deleteByQueryCommandsCumulative.increment(); boolean madeIt=false; try { Query q = getQuery(cmd); @@ -473,8 +473,8 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState } finally { if (!madeIt) { - numErrors.incrementAndGet(); - numErrorsCumulative.incrementAndGet(); + numErrors.increment(); + numErrorsCumulative.increment(); } } } @@ -482,7 +482,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState @Override public int mergeIndexes(MergeIndexesCommand cmd) throws IOException { - mergeIndexesCommands.incrementAndGet(); + mergeIndexesCommands.increment(); int rc; log.info("start " + cmd); @@ -545,7 +545,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState error=false; } finally { - if (error) numErrors.incrementAndGet(); + if (error) numErrors.increment(); } } @@ -557,10 +557,10 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState } if (cmd.optimize) { - optimizeCommands.incrementAndGet(); + optimizeCommands.increment(); } else { - commitCommands.incrementAndGet(); - if (cmd.expungeDeletes) expungeDeleteCommands.incrementAndGet(); + commitCommands.increment(); + if (cmd.expungeDeletes) expungeDeleteCommands.increment(); } Future[] waitSearcher = null; @@ -622,7 +622,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState } // SolrCore.verbose("writer.commit() end"); - numDocsPending.set(0); + numDocsPending.reset(); callPostCommitCallbacks(); } } finally { @@ -676,10 +676,10 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState solrCoreState.getCommitLock().unlock(); } - addCommands.set(0); - deleteByIdCommands.set(0); - deleteByQueryCommands.set(0); - if (error) numErrors.incrementAndGet(); + addCommands.reset(); + deleteByIdCommands.reset(); + deleteByQueryCommands.reset(); + if (error) numErrors.increment(); } // if we are supposed to wait for the searcher to be registered, then we should do it @@ -707,7 +707,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState throw new UnsupportedOperationException("Rollback is currently not supported in SolrCloud mode. (SOLR-4895)"); } - rollbackCommands.incrementAndGet(); + rollbackCommands.increment(); boolean error=true; @@ -727,13 +727,10 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState error=false; } finally { - addCommandsCumulative.set( - addCommandsCumulative.get() - addCommands.getAndSet( 0 ) ); - deleteByIdCommandsCumulative.set( - deleteByIdCommandsCumulative.get() - deleteByIdCommands.getAndSet( 0 ) ); - deleteByQueryCommandsCumulative.set( - deleteByQueryCommandsCumulative.get() - deleteByQueryCommands.getAndSet( 0 ) ); - if (error) numErrors.incrementAndGet(); + addCommandsCumulative.add(-addCommands.sumThenReset()); + deleteByIdCommandsCumulative.add(-deleteByIdCommands.sumThenReset()); + deleteByQueryCommandsCumulative.add(-deleteByQueryCommands.sumThenReset()); + if (error) numErrors.increment(); } } @@ -749,7 +746,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState commitTracker.close(); softCommitTracker.close(); - numDocsPending.set(0); + numDocsPending.reset(); } @@ -882,7 +879,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState @Override public NamedList getStatistics() { NamedList lst = new SimpleOrderedMap(); - lst.add("commits", commitCommands.get()); + lst.add("commits", commitCommands.longValue()); if (commitTracker.getDocsUpperBound() > 0) { lst.add("autocommit maxDocs", commitTracker.getDocsUpperBound()); } @@ -897,20 +894,20 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState lst.add("soft autocommit maxTime", "" + softCommitTracker.getTimeUpperBound() + "ms"); } lst.add("soft autocommits", softCommitTracker.getCommitCount()); - lst.add("optimizes", optimizeCommands.get()); - lst.add("rollbacks", rollbackCommands.get()); - lst.add("expungeDeletes", expungeDeleteCommands.get()); - lst.add("docsPending", numDocsPending.get()); + lst.add("optimizes", optimizeCommands.longValue()); + lst.add("rollbacks", rollbackCommands.longValue()); + lst.add("expungeDeletes", expungeDeleteCommands.longValue()); + lst.add("docsPending", numDocsPending.longValue()); // pset.size() not synchronized, but it should be fine to access. // lst.add("deletesPending", pset.size()); - lst.add("adds", addCommands.get()); - lst.add("deletesById", deleteByIdCommands.get()); - lst.add("deletesByQuery", deleteByQueryCommands.get()); - lst.add("errors", numErrors.get()); - lst.add("cumulative_adds", addCommandsCumulative.get()); - lst.add("cumulative_deletesById", deleteByIdCommandsCumulative.get()); - lst.add("cumulative_deletesByQuery", deleteByQueryCommandsCumulative.get()); - lst.add("cumulative_errors", numErrorsCumulative.get()); + lst.add("adds", addCommands.longValue()); + lst.add("deletesById", deleteByIdCommands.longValue()); + lst.add("deletesByQuery", deleteByQueryCommands.longValue()); + lst.add("errors", numErrors.longValue()); + lst.add("cumulative_adds", addCommandsCumulative.longValue()); + lst.add("cumulative_deletesById", deleteByIdCommandsCumulative.longValue()); + lst.add("cumulative_deletesByQuery", deleteByQueryCommandsCumulative.longValue()); + lst.add("cumulative_errors", numErrorsCumulative.longValue()); if (this.ulog != null) { lst.add("transaction_logs_total_size", ulog.getTotalLogsSize()); lst.add("transaction_logs_total_number", ulog.getTotalLogsNumber()); diff --git a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java index 89e286a7d67..873a697e49c 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java @@ -163,4 +163,6 @@ public abstract class SolrCoreState { super(s); } } + + public abstract Lock getRecoveryLock(); } diff --git a/solr/core/src/java/org/apache/solr/update/processor/CdcrUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/CdcrUpdateProcessor.java index 3b3fcb420ad..5bbc4a2a845 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/CdcrUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/CdcrUpdateProcessor.java @@ -96,11 +96,11 @@ public class CdcrUpdateProcessor extends DistributedUpdateProcessor { ModifiableSolrParams result = super.filterParams(params); if (params.get(CDCR_UPDATE) != null) { result.set(CDCR_UPDATE, ""); - if (params.get(DistributedUpdateProcessor.VERSION_FIELD) == null) { - log.warn("+++ cdcr.update but no version field, params are: " + params); - } else { - log.info("+++ cdcr.update version present, params are: " + params); - } +// if (params.get(DistributedUpdateProcessor.VERSION_FIELD) == null) { +// log.warn("+++ cdcr.update but no version field, params are: " + params); +// } else { +// log.info("+++ cdcr.update version present, params are: " + params); +// } result.set(DistributedUpdateProcessor.VERSION_FIELD, params.get(DistributedUpdateProcessor.VERSION_FIELD)); } diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java index 67c88ddb9f7..9b0e4dcfbb2 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java @@ -295,7 +295,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor { // this should always be used - see filterParams DistributedUpdateProcessorFactory.addParamToDistributedRequestWhitelist - (this.req, UpdateParams.UPDATE_CHAIN, TEST_DISTRIB_SKIP_SERVERS); + (this.req, UpdateParams.UPDATE_CHAIN, TEST_DISTRIB_SKIP_SERVERS, VERSION_FIELD); CoreDescriptor coreDesc = req.getCore().getCoreDescriptor(); diff --git a/solr/core/src/java/org/apache/solr/util/ConcurrentLRUCache.java b/solr/core/src/java/org/apache/solr/util/ConcurrentLRUCache.java index 3b6db53ae92..be14437fda1 100644 --- a/solr/core/src/java/org/apache/solr/util/ConcurrentLRUCache.java +++ b/solr/core/src/java/org/apache/solr/util/ConcurrentLRUCache.java @@ -28,6 +28,7 @@ import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantLock; import java.lang.invoke.MethodHandles; import java.lang.ref.WeakReference; @@ -90,7 +91,7 @@ public class ConcurrentLRUCache implements Cache { public V get(K key) { CacheEntry e = map.get(key); if (e == null) { - if (islive) stats.missCounter.incrementAndGet(); + if (islive) stats.missCounter.increment(); return null; } if (islive) e.lastAccessed = stats.accessCounter.incrementAndGet(); @@ -119,9 +120,9 @@ public class ConcurrentLRUCache implements Cache { currentSize = stats.size.get(); } if (islive) { - stats.putCounter.incrementAndGet(); + stats.putCounter.increment(); } else { - stats.nonLivePutCounter.incrementAndGet(); + stats.nonLivePutCounter.increment(); } // Check if we need to clear out old entries from the cache. @@ -172,7 +173,7 @@ public class ConcurrentLRUCache implements Cache { isCleaning = true; this.oldestEntry = oldestEntry; // volatile write to make isCleaning visible - long timeCurrent = stats.accessCounter.get(); + long timeCurrent = stats.accessCounter.longValue(); int sz = stats.size.get(); int numRemoved = 0; @@ -532,23 +533,23 @@ public class ConcurrentLRUCache implements Cache { public static class Stats { - private final AtomicLong accessCounter = new AtomicLong(0), - putCounter = new AtomicLong(0), - nonLivePutCounter = new AtomicLong(0), - missCounter = new AtomicLong(); + private final AtomicLong accessCounter = new AtomicLong(0); + private final LongAdder putCounter = new LongAdder(); + private final LongAdder nonLivePutCounter = new LongAdder(); + private final LongAdder missCounter = new LongAdder(); private final AtomicInteger size = new AtomicInteger(); private AtomicLong evictionCounter = new AtomicLong(); public long getCumulativeLookups() { - return (accessCounter.get() - putCounter.get() - nonLivePutCounter.get()) + missCounter.get(); + return (accessCounter.longValue() - putCounter.longValue() - nonLivePutCounter.longValue()) + missCounter.longValue(); } public long getCumulativeHits() { - return accessCounter.get() - putCounter.get() - nonLivePutCounter.get(); + return accessCounter.longValue() - putCounter.longValue() - nonLivePutCounter.longValue(); } public long getCumulativePuts() { - return putCounter.get(); + return putCounter.longValue(); } public long getCumulativeEvictions() { @@ -560,18 +561,18 @@ public class ConcurrentLRUCache implements Cache { } public long getCumulativeNonLivePuts() { - return nonLivePutCounter.get(); + return nonLivePutCounter.longValue(); } public long getCumulativeMisses() { - return missCounter.get(); + return missCounter.longValue(); } public void add(Stats other) { accessCounter.addAndGet(other.accessCounter.get()); - putCounter.addAndGet(other.putCounter.get()); - nonLivePutCounter.addAndGet(other.nonLivePutCounter.get()); - missCounter.addAndGet(other.missCounter.get()); + putCounter.add(other.putCounter.longValue()); + nonLivePutCounter.add(other.nonLivePutCounter.longValue()); + missCounter.add(other.missCounter.longValue()); evictionCounter.addAndGet(other.evictionCounter.get()); size.set(Math.max(size.get(), other.size.get())); } diff --git a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java index c12104f913e..374e901f65f 100644 --- a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java +++ b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java @@ -486,7 +486,7 @@ public class SolrPluginUtils { String qs = commands.size() >= 1 ? commands.get(0) : ""; try { - Query query = QParser.getParser(qs, null, req).getQuery(); + Query query = QParser.getParser(qs, req).getQuery(); // If the first non-query, non-filter command is a simple sort on an indexed field, then // we can use the Lucene sort ability. @@ -978,7 +978,7 @@ public class SolrPluginUtils { List out = new ArrayList<>(queries.length); for (String q : queries) { if (null != q && 0 != q.trim().length()) { - out.add(QParser.getParser(q, null, req).getQuery()); + out.add(QParser.getParser(q, req).getQuery()); } } return out; diff --git a/solr/core/src/java/org/apache/solr/util/TestInjection.java b/solr/core/src/java/org/apache/solr/util/TestInjection.java index cc3f85d4472..03de74d02f1 100644 --- a/solr/core/src/java/org/apache/solr/util/TestInjection.java +++ b/solr/core/src/java/org/apache/solr/util/TestInjection.java @@ -17,6 +17,7 @@ package org.apache.solr.util; import java.lang.invoke.MethodHandles; +import java.lang.reflect.Method; import java.util.Collections; import java.util.HashSet; import java.util.Random; @@ -39,6 +40,11 @@ import org.slf4j.LoggerFactory; * Allows random faults to be injected in running code during test runs. * * Set static strings to "true" or "false" or "true:60" for true 60% of the time. + * + * All methods are No-Ops unless LuceneTestCase is loadable via the ClassLoader used + * to load this class. LuceneTestCase.random() is used as the source of all entropy. + * + * @lucene.internal */ public class TestInjection { @@ -53,16 +59,42 @@ public class TestInjection { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final Pattern ENABLED_PERCENT = Pattern.compile("(true|false)(?:\\:(\\d+))?$", Pattern.CASE_INSENSITIVE); - private static final Random RANDOM; + + private static final String LUCENE_TEST_CASE_FQN = "org.apache.lucene.util.LuceneTestCase"; + + /** + * If null, then we are not being run as part of a test, and all TestInjection events should be No-Ops. + * If non-null, then this class should be used for accessing random entropy + * @see #random + */ + private static final Class LUCENE_TEST_CASE; static { - // We try to make things reproducible in the context of our tests by initializing the random instance - // based on the current seed - String seed = System.getProperty("tests.seed"); - if (seed == null) { - RANDOM = new Random(); + Class nonFinalTemp = null; + try { + ClassLoader classLoader = MethodHandles.lookup().lookupClass().getClassLoader(); + nonFinalTemp = classLoader.loadClass(LUCENE_TEST_CASE_FQN); + } catch (ClassNotFoundException e) { + log.debug("TestInjection methods will all be No-Ops since LuceneTestCase not found"); + } + LUCENE_TEST_CASE = nonFinalTemp; + } + + /** + * Returns a random to be used by the current thread if available, otherwise + * returns null. + * @see #LUCENE_TEST_CASE + */ + static Random random() { // non-private for testing + if (null == LUCENE_TEST_CASE) { + return null; } else { - RANDOM = new Random(seed.hashCode()); + try { + Method randomMethod = LUCENE_TEST_CASE.getMethod("random"); + return (Random) randomMethod.invoke(null); + } catch (Exception e) { + throw new IllegalStateException("Unable to use reflection to invoke LuceneTestCase.random()", e); + } } } @@ -100,11 +132,14 @@ public class TestInjection { public static boolean injectRandomDelayInCoreCreation() { if (randomDelayInCoreCreation != null) { + Random rand = random(); + if (null == rand) return true; + Pair pair = parseValue(randomDelayInCoreCreation); boolean enabled = pair.first(); int chanceIn100 = pair.second(); - if (enabled && RANDOM.nextInt(100) >= (100 - chanceIn100)) { - int delay = RANDOM.nextInt(randomDelayMaxInCoreCreationInSec); + if (enabled && rand.nextInt(100) >= (100 - chanceIn100)) { + int delay = rand.nextInt(randomDelayMaxInCoreCreationInSec); log.info("Inject random core creation delay of {}s", delay); try { Thread.sleep(delay * 1000); @@ -118,11 +153,14 @@ public class TestInjection { public static boolean injectNonGracefullClose(CoreContainer cc) { if (cc.isShutDown() && nonGracefullClose != null) { + Random rand = random(); + if (null == rand) return true; + Pair pair = parseValue(nonGracefullClose); boolean enabled = pair.first(); int chanceIn100 = pair.second(); - if (enabled && RANDOM.nextInt(100) >= (100 - chanceIn100)) { - if (RANDOM.nextBoolean()) { + if (enabled && rand.nextInt(100) >= (100 - chanceIn100)) { + if (rand.nextBoolean()) { throw new TestShutdownFailError("Test exception for non graceful close"); } else { @@ -135,7 +173,9 @@ public class TestInjection { // we should only need to do it once try { - Thread.sleep(RANDOM.nextInt(1000)); + // call random() again to get the correct one for this thread + Random taskRand = random(); + Thread.sleep(taskRand.nextInt(1000)); } catch (InterruptedException e) { } @@ -147,7 +187,7 @@ public class TestInjection { }; Timer timer = new Timer(); timers.add(timer); - timer.schedule(task, RANDOM.nextInt(500)); + timer.schedule(task, rand.nextInt(500)); } } } @@ -156,10 +196,13 @@ public class TestInjection { public static boolean injectFailReplicaRequests() { if (failReplicaRequests != null) { + Random rand = random(); + if (null == rand) return true; + Pair pair = parseValue(failReplicaRequests); boolean enabled = pair.first(); int chanceIn100 = pair.second(); - if (enabled && RANDOM.nextInt(100) >= (100 - chanceIn100)) { + if (enabled && rand.nextInt(100) >= (100 - chanceIn100)) { throw new SolrException(ErrorCode.SERVER_ERROR, "Random test update fail"); } } @@ -169,10 +212,13 @@ public class TestInjection { public static boolean injectFailUpdateRequests() { if (failUpdateRequests != null) { + Random rand = random(); + if (null == rand) return true; + Pair pair = parseValue(failUpdateRequests); boolean enabled = pair.first(); int chanceIn100 = pair.second(); - if (enabled && RANDOM.nextInt(100) >= (100 - chanceIn100)) { + if (enabled && rand.nextInt(100) >= (100 - chanceIn100)) { throw new SolrException(ErrorCode.SERVER_ERROR, "Random test update fail"); } } @@ -182,10 +228,13 @@ public class TestInjection { public static boolean injectNonExistentCoreExceptionAfterUnload(String cname) { if (nonExistentCoreExceptionAfterUnload != null) { + Random rand = random(); + if (null == rand) return true; + Pair pair = parseValue(nonExistentCoreExceptionAfterUnload); boolean enabled = pair.first(); int chanceIn100 = pair.second(); - if (enabled && RANDOM.nextInt(100) >= (100 - chanceIn100)) { + if (enabled && rand.nextInt(100) >= (100 - chanceIn100)) { throw new NonExistentCoreException("Core not found to unload: " + cname); } } @@ -195,11 +244,14 @@ public class TestInjection { public static boolean injectUpdateLogReplayRandomPause() { if (updateLogReplayRandomPause != null) { + Random rand = random(); + if (null == rand) return true; + Pair pair = parseValue(updateLogReplayRandomPause); boolean enabled = pair.first(); int chanceIn100 = pair.second(); - if (enabled && RANDOM.nextInt(100) >= (100 - chanceIn100)) { - long rndTime = RANDOM.nextInt(1000); + if (enabled && rand.nextInt(100) >= (100 - chanceIn100)) { + long rndTime = rand.nextInt(1000); log.info("inject random log replay delay of {}ms", rndTime); try { Thread.sleep(rndTime); @@ -214,11 +266,14 @@ public class TestInjection { public static boolean injectUpdateRandomPause() { if (updateRandomPause != null) { + Random rand = random(); + if (null == rand) return true; + Pair pair = parseValue(updateRandomPause); boolean enabled = pair.first(); int chanceIn100 = pair.second(); - if (enabled && RANDOM.nextInt(100) >= (100 - chanceIn100)) { - long rndTime = RANDOM.nextInt(1000); + if (enabled && rand.nextInt(100) >= (100 - chanceIn100)) { + long rndTime = rand.nextInt(1000); log.info("inject random update delay of {}ms", rndTime); try { Thread.sleep(rndTime); diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-psuedo-fields.xml b/solr/core/src/test-files/solr/collection1/conf/schema-psuedo-fields.xml index 6cb006a766b..20f2d2dd299 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-psuedo-fields.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-psuedo-fields.xml @@ -23,6 +23,7 @@ + + + + + + + + + + + + id + diff --git a/solr/core/src/test-files/solr/configsets/cdcr-source-disabled/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cdcr-source-disabled/solrconfig.xml new file mode 100644 index 00000000000..e63d9a6506c --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/cdcr-source-disabled/solrconfig.xml @@ -0,0 +1,60 @@ + + + + + + + + + ${solr.data.dir:} + + + + + ${tests.luceneMatchVersion:LATEST} + + + + ${solr.commitwithin.softcommit:true} + + + + ${solr.ulog.dir:} + + + + + + explicit + true + text + + + + + diff --git a/solr/core/src/test-files/solr/configsets/cdcr-source/schema.xml b/solr/core/src/test-files/solr/configsets/cdcr-source/schema.xml new file mode 100644 index 00000000000..2897315141c --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/cdcr-source/schema.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + id + diff --git a/solr/core/src/test-files/solr/configsets/cdcr-source/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cdcr-source/solrconfig.xml new file mode 100644 index 00000000000..f2528c3ad10 --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/cdcr-source/solrconfig.xml @@ -0,0 +1,76 @@ + + + + + + + + + ${solr.data.dir:} + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + + + + ${cdcr.target.zkHost} + cdcr-source + cdcr-target + + + 1 + 1000 + 1000 + + + 1000 + + + + + + ${solr.ulog.dir:} + + + + + + + + + cdcr-processor-chain + + + + diff --git a/solr/core/src/test-files/solr/configsets/cdcr-target/schema.xml b/solr/core/src/test-files/solr/configsets/cdcr-target/schema.xml new file mode 100644 index 00000000000..2897315141c --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/cdcr-target/schema.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + id + diff --git a/solr/core/src/test-files/solr/configsets/cdcr-target/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cdcr-target/solrconfig.xml new file mode 100644 index 00000000000..ef24fa49a46 --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/cdcr-target/solrconfig.xml @@ -0,0 +1,63 @@ + + + + + + + + + ${solr.data.dir:} + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + + + + + + + ${solr.ulog.dir:} + + + + + + + + + cdcr-processor-chain + + + + diff --git a/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java index 289a75f9a5a..269a2c55089 100644 --- a/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java +++ b/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java @@ -131,7 +131,7 @@ public class TestReversedWildcardFilterFactory extends SolrTestCaseJ4 { "//result[@numFound=1]"); SolrQueryRequest req = req(); - QParser qparser = QParser.getParser("id:1", "lucene", req); + QParser qparser = QParser.getParser("id:1", req); SolrQueryParser parserTwo = new SolrQueryParser(qparser, "two"); assertTrue(parserTwo.getAllowLeadingWildcard()); diff --git a/solr/core/src/test/org/apache/solr/cloud/BaseCdcrDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BaseCdcrDistributedZkTest.java index c53532f656e..3d758e046fe 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BaseCdcrDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BaseCdcrDistributedZkTest.java @@ -28,6 +28,7 @@ import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; @@ -49,12 +50,14 @@ import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.SolrCore; import org.apache.solr.handler.CdcrParams; +import org.apache.solr.util.TimeOut; import org.apache.zookeeper.CreateMode; import org.junit.After; import org.junit.AfterClass; @@ -68,6 +71,8 @@ import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES; import static org.apache.solr.cloud.OverseerCollectionMessageHandler.SHARDS_PROP; import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE; import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; +import static org.apache.solr.handler.admin.CoreAdminHandler.COMPLETED; +import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE_STATUS; /** *

@@ -667,6 +672,14 @@ public class BaseCdcrDistributedZkTest extends AbstractDistribZkTestBase { } } + List oldRunners = this.cloudJettys.putIfAbsent(collection, cloudJettys); + if (oldRunners != null) { + // must close resources for the old entries + for (CloudJettyRunner oldRunner : oldRunners) { + IOUtils.closeQuietly(oldRunner.client); + } + } + this.cloudJettys.put(collection, cloudJettys); this.shardToJetty.put(collection, shardToJetty); this.shardToLeaderJetty.put(collection, shardToLeaderJetty); @@ -754,6 +767,18 @@ public class BaseCdcrDistributedZkTest extends AbstractDistribZkTestBase { } } + protected void waitForBootstrapToComplete(String collectionName, String shardId) throws Exception { + NamedList rsp;// we need to wait until bootstrap is complete otherwise the replicator thread will never start + TimeOut timeOut = new TimeOut(60, TimeUnit.SECONDS); + while (!timeOut.hasTimedOut()) { + rsp = invokeCdcrAction(shardToLeaderJetty.get(collectionName).get(shardId), CdcrParams.CdcrAction.BOOTSTRAP_STATUS); + if (rsp.get(RESPONSE_STATUS).toString().equals(COMPLETED)) { + break; + } + Thread.sleep(1000); + } + } + protected void waitForReplicationToComplete(String collectionName, String shardId) throws Exception { int cnt = 15; while (cnt > 0) { diff --git a/solr/core/src/test/org/apache/solr/cloud/CdcrBootstrapTest.java b/solr/core/src/test/org/apache/solr/cloud/CdcrBootstrapTest.java new file mode 100644 index 00000000000..1efdc6a733c --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/CdcrBootstrapTest.java @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.AbstractUpdateRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.handler.CdcrParams; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CdcrBootstrapTest extends SolrTestCaseJ4 { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * Starts a source cluster with no CDCR configuration, indexes enough documents such that + * the at least one old tlog is closed and thrown away so that the source cluster does not have + * all updates available in tlogs only. + *

+ * Then we start a target cluster with CDCR configuration and we change the source cluster configuration + * to use CDCR (i.e. CdcrUpdateLog, CdcrRequestHandler and CdcrUpdateProcessor) and restart it. + *

+ * We test that all updates eventually make it to the target cluster and that the collectioncheckpoint + * call returns the same version as the last update indexed on the source. + */ + @Test + public void testConvertClusterToCdcrAndBootstrap() throws Exception { + // start the target first so that we know its zkhost + MiniSolrCloudCluster target = new MiniSolrCloudCluster(1, createTempDir("cdcr-target"), buildJettyConfig("/solr")); + try { + target.waitForAllNodes(30); + System.out.println("Target zkHost = " + target.getZkServer().getZkAddress()); + System.setProperty("cdcr.target.zkHost", target.getZkServer().getZkAddress()); + + // start a cluster with no cdcr + MiniSolrCloudCluster source = new MiniSolrCloudCluster(1, createTempDir("cdcr-source"), buildJettyConfig("/solr")); + try { + source.waitForAllNodes(30); + final File configDir = getFile("solr").toPath().resolve("configsets/cdcr-source-disabled").toFile(); + System.out.println("config dir absolute path = " + configDir.getAbsolutePath()); + source.uploadConfigDir(configDir, "cdcr-source"); + + // create a collection with the cdcr-source-disabled configset + Map collectionProperties = new HashMap<>(); + // todo investigate why this is necessary??? because by default it selects a ram directory which deletes the tlogs on reloads? + collectionProperties.putIfAbsent("solr.directoryFactory", "solr.StandardDirectoryFactory"); + source.createCollection("cdcr-source", 1, 1, "cdcr-source", collectionProperties); + source.getSolrClient().getZkStateReader().forceUpdateCollection("cdcr-source"); + AbstractDistribZkTestBase.waitForRecoveriesToFinish("cdcr-source", source.getSolrClient().getZkStateReader(), true, true, 330); + + // index 10000 docs with a hard commit every 1000 documents + CloudSolrClient sourceSolrClient = source.getSolrClient(); + sourceSolrClient.setDefaultCollection("cdcr-source"); + int numDocs = 0; + for (int k = 0; k < 100; k++) { + UpdateRequest req = new UpdateRequest(); + for (; numDocs < (k + 1) * 100; numDocs++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", "source_" + numDocs); + doc.addField("xyz", numDocs); + req.add(doc); + } + req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); + System.out.println("Adding 100 docs with commit=true, numDocs=" + numDocs); + req.process(sourceSolrClient); + } + + QueryResponse response = sourceSolrClient.query(new SolrQuery("*:*")); + assertEquals("", numDocs, response.getResults().getNumFound()); + + // lets find and keep the maximum version assigned by source cluster across all our updates + long maxVersion = Long.MIN_VALUE; + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CommonParams.QT, "/get"); + params.set("getVersions", numDocs); + response = sourceSolrClient.query(params); + List versions = (List) response.getResponse().get("versions"); + for (Long version : versions) { + maxVersion = Math.max(maxVersion, version); + } + +// upload the cdcr-enabled config and restart source cluster + final File cdcrEnabledSourceConfigDir = getFile("solr").toPath().resolve("configsets/cdcr-source").toFile(); + source.uploadConfigDir(cdcrEnabledSourceConfigDir, "cdcr-source"); + JettySolrRunner runner = source.stopJettySolrRunner(0); + source.startJettySolrRunner(runner); + assertTrue(runner.isRunning()); + AbstractDistribZkTestBase.waitForRecoveriesToFinish("cdcr-source", source.getSolrClient().getZkStateReader(), true, true, 330); + + response = sourceSolrClient.query(new SolrQuery("*:*")); + assertEquals("Document mismatch on source after restart", numDocs, response.getResults().getNumFound()); + + // setup the target cluster + final File targetConfigDir = getFile("solr").toPath().resolve("configsets/cdcr-target").toFile(); + target.uploadConfigDir(targetConfigDir, "cdcr-target"); + target.createCollection("cdcr-target", 1, 1, "cdcr-target", Collections.emptyMap()); + target.getSolrClient().getZkStateReader().forceUpdateCollection("cdcr-target"); + AbstractDistribZkTestBase.waitForRecoveriesToFinish("cdcr-target", target.getSolrClient().getZkStateReader(), true, true, 330); + CloudSolrClient targetSolrClient = target.getSolrClient(); + targetSolrClient.setDefaultCollection("cdcr-target"); + Thread.sleep(1000); + + cdcrStart(targetSolrClient); + cdcrStart(sourceSolrClient); + + response = getCdcrQueue(sourceSolrClient); + System.out.println("Cdcr queue response: " + response.getResponse()); + long foundDocs = waitForTargetToSync(numDocs, targetSolrClient); + assertEquals("Document mismatch on target after sync", numDocs, foundDocs); + + params = new ModifiableSolrParams(); + params.set(CommonParams.ACTION, CdcrParams.CdcrAction.COLLECTIONCHECKPOINT.toString()); + params.set(CommonParams.QT, "/cdcr"); + response = targetSolrClient.query(params); + Long checkpoint = (Long) response.getResponse().get(CdcrParams.CHECKPOINT); + assertNotNull(checkpoint); + assertEquals("COLLECTIONCHECKPOINT from target cluster should have returned the maximum " + + "version across all updates made to source", maxVersion, checkpoint.longValue()); + } finally { + source.shutdown(); + } + } finally { + target.shutdown(); + } + } + + /** + * This test start cdcr source, adds data,starts target cluster, verifies replication, + * stops cdcr replication and buffering, adds more data, re-enables cdcr and verify replication + */ + public void testBootstrapWithSourceCluster() throws Exception { + // start the target first so that we know its zkhost + MiniSolrCloudCluster target = new MiniSolrCloudCluster(1, createTempDir("cdcr-target"), buildJettyConfig("/solr")); + try { + target.waitForAllNodes(30); + System.out.println("Target zkHost = " + target.getZkServer().getZkAddress()); + System.setProperty("cdcr.target.zkHost", target.getZkServer().getZkAddress()); + + MiniSolrCloudCluster source = new MiniSolrCloudCluster(1, createTempDir("cdcr-source"), buildJettyConfig("/solr")); + try { + source.waitForAllNodes(30); + final File configDir = getFile("solr").toPath().resolve("configsets/cdcr-source").toFile(); + System.out.println("config dir absolute path = " + configDir.getAbsolutePath()); + source.uploadConfigDir(configDir, "cdcr-source"); + + Map collectionProperties = new HashMap<>(); + // todo investigate why this is necessary??? + collectionProperties.putIfAbsent("solr.directoryFactory", "solr.StandardDirectoryFactory"); + source.createCollection("cdcr-source", 1, 1, "cdcr-source", collectionProperties); + source.getSolrClient().getZkStateReader().forceUpdateCollection("cdcr-source"); + AbstractDistribZkTestBase.waitForRecoveriesToFinish("cdcr-source", source.getSolrClient().getZkStateReader(), true, true, 330); + + // index 10000 docs with a hard commit every 1000 documents + CloudSolrClient sourceSolrClient = source.getSolrClient(); + sourceSolrClient.setDefaultCollection("cdcr-source"); + int numDocs = 0; + for (int k = 0; k < 100; k++) { + UpdateRequest req = new UpdateRequest(); + for (; numDocs < (k + 1) * 100; numDocs++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", "source_" + numDocs); + doc.addField("xyz", numDocs); + req.add(doc); + } + req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); + System.out.println("Adding 100 docs with commit=true, numDocs=" + numDocs); + req.process(sourceSolrClient); + } + + QueryResponse response = sourceSolrClient.query(new SolrQuery("*:*")); + assertEquals("", numDocs, response.getResults().getNumFound()); + + // setup the target cluster + final File targetConfigDir = getFile("solr").toPath().resolve("configsets/cdcr-target").toFile(); + target.uploadConfigDir(targetConfigDir, "cdcr-target"); + target.createCollection("cdcr-target", 1, 1, "cdcr-target", Collections.emptyMap()); + target.getSolrClient().getZkStateReader().forceUpdateCollection("cdcr-target"); + AbstractDistribZkTestBase.waitForRecoveriesToFinish("cdcr-target", target.getSolrClient().getZkStateReader(), true, true, 330); + CloudSolrClient targetSolrClient = target.getSolrClient(); + targetSolrClient.setDefaultCollection("cdcr-target"); + + cdcrStart(targetSolrClient); + cdcrStart(sourceSolrClient); + + response = getCdcrQueue(sourceSolrClient); + System.out.println("Cdcr queue response: " + response.getResponse()); + long foundDocs = waitForTargetToSync(numDocs, targetSolrClient); + assertEquals("Document mismatch on target after sync", numDocs, foundDocs); + + cdcrStop(sourceSolrClient); + cdcrDisableBuffer(sourceSolrClient); + + int c = 0; + for (int k = 0; k < 100; k++) { + UpdateRequest req = new UpdateRequest(); + for (; c < (k + 1) * 100; c++, numDocs++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", "source_" + numDocs); + doc.addField("xyz", numDocs); + req.add(doc); + } + req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); + System.out.println("Adding 100 docs with commit=true, numDocs=" + numDocs); + req.process(sourceSolrClient); + } + + response = sourceSolrClient.query(new SolrQuery("*:*")); + assertEquals("", numDocs, response.getResults().getNumFound()); + + cdcrStart(sourceSolrClient); + cdcrEnableBuffer(sourceSolrClient); + + foundDocs = waitForTargetToSync(numDocs, targetSolrClient); + assertEquals("Document mismatch on target after sync", numDocs, foundDocs); + + } finally { + source.shutdown(); + } + } finally { + target.shutdown(); + } + } + + public void testBootstrapWithContinousIndexingOnSourceCluster() throws Exception { + // start the target first so that we know its zkhost + MiniSolrCloudCluster target = new MiniSolrCloudCluster(1, createTempDir("cdcr-target"), buildJettyConfig("/solr")); + target.waitForAllNodes(30); + try { + System.out.println("Target zkHost = " + target.getZkServer().getZkAddress()); + System.setProperty("cdcr.target.zkHost", target.getZkServer().getZkAddress()); + + MiniSolrCloudCluster source = new MiniSolrCloudCluster(1, createTempDir("cdcr-source"), buildJettyConfig("/solr")); + try { + source.waitForAllNodes(30); + final File configDir = getFile("solr").toPath().resolve("configsets/cdcr-source").toFile(); + System.out.println("config dir absolute path = " + configDir.getAbsolutePath()); + source.uploadConfigDir(configDir, "cdcr-source"); + + Map collectionProperties = new HashMap<>(); + // todo investigate why this is necessary??? + collectionProperties.putIfAbsent("solr.directoryFactory", "solr.StandardDirectoryFactory"); + source.createCollection("cdcr-source", 1, 1, "cdcr-source", collectionProperties); + source.getSolrClient().getZkStateReader().forceUpdateCollection("cdcr-source"); + AbstractDistribZkTestBase.waitForRecoveriesToFinish("cdcr-source", source.getSolrClient().getZkStateReader(), true, true, 330); + + // index 10000 docs with a hard commit every 1000 documents + CloudSolrClient sourceSolrClient = source.getSolrClient(); + sourceSolrClient.setDefaultCollection("cdcr-source"); + int numDocs = 0; + for (int k = 0; k < 100; k++) { + UpdateRequest req = new UpdateRequest(); + for (; numDocs < (k + 1) * 100; numDocs++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", "source_" + numDocs); + doc.addField("xyz", numDocs); + req.add(doc); + } + req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); + System.out.println("Adding 100 docs with commit=true, numDocs=" + numDocs); + req.process(sourceSolrClient); + } + + QueryResponse response = sourceSolrClient.query(new SolrQuery("*:*")); + assertEquals("", numDocs, response.getResults().getNumFound()); + + // setup the target cluster + final File targetConfigDir = getFile("solr").toPath().resolve("configsets/cdcr-target").toFile(); + target.uploadConfigDir(targetConfigDir, "cdcr-target"); + target.createCollection("cdcr-target", 1, 1, "cdcr-target", Collections.emptyMap()); + target.getSolrClient().getZkStateReader().forceUpdateCollection("cdcr-target"); + AbstractDistribZkTestBase.waitForRecoveriesToFinish("cdcr-target", target.getSolrClient().getZkStateReader(), true, true, 330); + CloudSolrClient targetSolrClient = target.getSolrClient(); + targetSolrClient.setDefaultCollection("cdcr-target"); + Thread.sleep(1000); + + cdcrStart(targetSolrClient); + cdcrStart(sourceSolrClient); + + int c = 0; + for (int k = 0; k < 100; k++) { + UpdateRequest req = new UpdateRequest(); + for (; c < (k + 1) * 100; c++, numDocs++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", "source_" + numDocs); + doc.addField("xyz", numDocs); + req.add(doc); + } + req.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); + System.out.println("Adding 100 docs with commit=true, numDocs=" + numDocs); + req.process(sourceSolrClient); + } + + response = sourceSolrClient.query(new SolrQuery("*:*")); + assertEquals("", numDocs, response.getResults().getNumFound()); + + response = getCdcrQueue(sourceSolrClient); + System.out.println("Cdcr queue response: " + response.getResponse()); + long foundDocs = waitForTargetToSync(numDocs, targetSolrClient); + assertEquals("Document mismatch on target after sync", numDocs, foundDocs); + + } finally { + source.shutdown(); + } + } finally { + target.shutdown(); + } + } + + private long waitForTargetToSync(int numDocs, CloudSolrClient targetSolrClient) throws SolrServerException, IOException, InterruptedException { + long start = System.nanoTime(); + QueryResponse response = null; + while (System.nanoTime() - start <= TimeUnit.NANOSECONDS.convert(120, TimeUnit.SECONDS)) { + try { + targetSolrClient.commit(); + response = targetSolrClient.query(new SolrQuery("*:*")); + if (response.getResults().getNumFound() == numDocs) { + break; + } + } catch (Exception e) { + log.warn("Exception trying to commit on target. This is expected and safe to ignore.", e); + } + Thread.sleep(1000); + } + return response != null ? response.getResults().getNumFound() : 0; + } + + + private void cdcrStart(CloudSolrClient client) throws SolrServerException, IOException { + QueryResponse response = invokeCdcrAction(client, CdcrParams.CdcrAction.START); + assertEquals("started", ((NamedList) response.getResponse().get("status")).get("process")); + } + + private void cdcrStop(CloudSolrClient client) throws SolrServerException, IOException { + QueryResponse response = invokeCdcrAction(client, CdcrParams.CdcrAction.STOP); + assertEquals("stopped", ((NamedList) response.getResponse().get("status")).get("process")); + } + + private void cdcrEnableBuffer(CloudSolrClient client) throws IOException, SolrServerException { + QueryResponse response = invokeCdcrAction(client, CdcrParams.CdcrAction.ENABLEBUFFER); + assertEquals("enabled", ((NamedList) response.getResponse().get("status")).get("buffer")); + } + + private void cdcrDisableBuffer(CloudSolrClient client) throws IOException, SolrServerException { + QueryResponse response = invokeCdcrAction(client, CdcrParams.CdcrAction.DISABLEBUFFER); + assertEquals("disabled", ((NamedList) response.getResponse().get("status")).get("buffer")); + } + + private QueryResponse invokeCdcrAction(CloudSolrClient client, CdcrParams.CdcrAction action) throws IOException, SolrServerException { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CommonParams.QT, "/cdcr"); + params.set(CommonParams.ACTION, action.toLower()); + return client.query(params); + } + + private QueryResponse getCdcrQueue(CloudSolrClient client) throws SolrServerException, IOException { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CommonParams.QT, "/cdcr"); + params.set(CommonParams.ACTION, CdcrParams.QUEUES); + return client.query(params); + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/CdcrReplicationDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/CdcrReplicationDistributedZkTest.java index 3478df962fb..3ba61866496 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CdcrReplicationDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CdcrReplicationDistributedZkTest.java @@ -103,6 +103,11 @@ public class CdcrReplicationDistributedZkTest extends BaseCdcrDistributedZkTest // check status this.assertState(SOURCE_COLLECTION, CdcrParams.ProcessState.STARTED, CdcrParams.BufferState.ENABLED); + this.waitForBootstrapToComplete(TARGET_COLLECTION, SHARD2); + + // sleep for a bit to ensure that replicator threads are started + Thread.sleep(3000); + // Kill all the servers of the target this.deleteCollection(TARGET_COLLECTION); @@ -156,6 +161,9 @@ public class CdcrReplicationDistributedZkTest extends BaseCdcrDistributedZkTest this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START); this.waitForCdcrStateReplication(SOURCE_COLLECTION); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2); + this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1); this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2); @@ -182,6 +190,9 @@ public class CdcrReplicationDistributedZkTest extends BaseCdcrDistributedZkTest this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START); this.waitForCdcrStateReplication(SOURCE_COLLECTION); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2); + this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1); this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2); @@ -203,6 +214,9 @@ public class CdcrReplicationDistributedZkTest extends BaseCdcrDistributedZkTest this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START); this.waitForCdcrStateReplication(SOURCE_COLLECTION); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2); + log.info("Indexing 10 documents"); int start = 0; @@ -244,6 +258,9 @@ public class CdcrReplicationDistributedZkTest extends BaseCdcrDistributedZkTest log.info("Waiting for replication"); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2); + this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1); this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2); @@ -267,6 +284,9 @@ public class CdcrReplicationDistributedZkTest extends BaseCdcrDistributedZkTest this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START); this.waitForCdcrStateReplication(SOURCE_COLLECTION); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2); + log.info("Indexing 10 documents"); int start = 0; @@ -349,6 +369,9 @@ public class CdcrReplicationDistributedZkTest extends BaseCdcrDistributedZkTest index(SOURCE_COLLECTION, getDoc(id, Integer.toString(i))); } + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2); + // wait a bit for the replication to complete this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1); this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2); @@ -495,6 +518,8 @@ public class CdcrReplicationDistributedZkTest extends BaseCdcrDistributedZkTest // Start CDCR this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START); this.waitForCdcrStateReplication(SOURCE_COLLECTION); + this.waitForBootstrapToComplete(TARGET_COLLECTION, SHARD1); + this.waitForBootstrapToComplete(TARGET_COLLECTION, SHARD2); // wait a bit for the replication to complete this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1); @@ -526,6 +551,9 @@ public class CdcrReplicationDistributedZkTest extends BaseCdcrDistributedZkTest assertNumDocs(128, SOURCE_COLLECTION); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2); + this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1); this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2); @@ -553,6 +581,9 @@ public class CdcrReplicationDistributedZkTest extends BaseCdcrDistributedZkTest this.invokeCdcrAction(shardToLeaderJetty.get(SOURCE_COLLECTION).get(SHARD1), CdcrParams.CdcrAction.START); this.waitForCdcrStateReplication(SOURCE_COLLECTION); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD1); + this.waitForBootstrapToComplete(SOURCE_COLLECTION, SHARD2); + // wait a bit for the replication to complete this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD1); this.waitForReplicationToComplete(SOURCE_COLLECTION, SHARD2); diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java index 190db573a50..7e840daf2eb 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java @@ -16,6 +16,12 @@ */ package org.apache.solr.cloud; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.common.SolrInputDocument; @@ -36,12 +42,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - /** * Test split phase that occurs when a Collection API split call is made. */ @@ -254,6 +254,7 @@ public class ChaosMonkeyShardSplitTest extends ShardSplitTest { address.replaceAll("/", "_")); overseerElector.setup(ec); overseerElector.joinElection(ec, false); + reader.close(); return zkClient; } diff --git a/solr/core/src/test/org/apache/solr/cloud/KerberosTestUtil.java b/solr/core/src/test/org/apache/solr/cloud/KerberosTestServices.java similarity index 56% rename from solr/core/src/test/org/apache/solr/cloud/KerberosTestUtil.java rename to solr/core/src/test/org/apache/solr/cloud/KerberosTestServices.java index 7f544ef8fef..6295dddacbd 100644 --- a/solr/core/src/test/org/apache/solr/cloud/KerberosTestUtil.java +++ b/solr/core/src/test/org/apache/solr/cloud/KerberosTestServices.java @@ -16,6 +16,8 @@ */ package org.apache.solr.cloud; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; import java.io.File; import java.util.Arrays; import java.util.HashMap; @@ -24,18 +26,57 @@ import java.util.Locale; import java.util.Map; import java.util.Properties; -import javax.security.auth.login.AppConfigurationEntry; -import javax.security.auth.login.Configuration; - +import com.google.common.base.Preconditions; import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder; -public class KerberosTestUtil { +public class KerberosTestServices { + + private MiniKdc kdc; + private JaasConfiguration jaasConfiguration; + private Configuration savedConfig; + private Locale savedLocale; + + private KerberosTestServices(MiniKdc kdc, + JaasConfiguration jaasConfiguration, + Configuration savedConfig, + Locale savedLocale) { + this.kdc = kdc; + this.jaasConfiguration = jaasConfiguration; + this.savedConfig = savedConfig; + this.savedLocale = savedLocale; + } + + public MiniKdc getKdc() { + return kdc; + } + + public void start() throws Exception { + if (brokenLanguagesWithMiniKdc.contains(Locale.getDefault().getLanguage())) { + Locale.setDefault(Locale.US); + } + + if (kdc != null) kdc.start(); + Configuration.setConfiguration(jaasConfiguration); + Krb5HttpClientBuilder.regenerateJaasConfiguration(); + } + + public void stop() { + if (kdc != null) kdc.stop(); + Configuration.setConfiguration(savedConfig); + Krb5HttpClientBuilder.regenerateJaasConfiguration(); + Locale.setDefault(savedLocale); + } + + public static Builder builder() { + return new Builder(); + } /** * Returns a MiniKdc that can be used for creating kerberos principals * and keytabs. Caller is responsible for starting/stopping the kdc. */ - public static MiniKdc getKdc(File workDir) throws Exception { + private static MiniKdc getKdc(File workDir) throws Exception { Properties conf = MiniKdc.createConf(); return new MiniKdc(conf, workDir); } @@ -44,7 +85,7 @@ public class KerberosTestUtil { * Programmatic version of a jaas.conf file suitable for connecting * to a SASL-configured zookeeper. */ - public static class JaasConfiguration extends Configuration { + private static class JaasConfiguration extends Configuration { private static AppConfigurationEntry[] clientEntry; private static AppConfigurationEntry[] serverEntry; @@ -60,7 +101,7 @@ public class KerberosTestUtil { * @param serverKeytab The location of the keytab with the serverPrincipal */ public JaasConfiguration(String clientPrincipal, File clientKeytab, - String serverPrincipal, File serverKeytab) { + String serverPrincipal, File serverKeytab) { Map clientOptions = new HashMap(); clientOptions.put("principal", clientPrincipal); clientOptions.put("keyTab", clientKeytab.getAbsolutePath()); @@ -73,9 +114,9 @@ public class KerberosTestUtil { clientOptions.put("debug", "true"); } clientEntry = new AppConfigurationEntry[]{ - new AppConfigurationEntry(getKrb5LoginModuleName(), - AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, - clientOptions)}; + new AppConfigurationEntry(getKrb5LoginModuleName(), + AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, + clientOptions)}; if(serverPrincipal!=null && serverKeytab!=null) { Map serverOptions = new HashMap(clientOptions); serverOptions.put("principal", serverPrincipal); @@ -88,9 +129,9 @@ public class KerberosTestUtil { } /** - * Add an entry to the jaas configuration with the passed in principal and keytab, + * Add an entry to the jaas configuration with the passed in principal and keytab, * along with the app name. - * + * * @param principal The principal * @param keytab The keytab containing credentials for the principal * @param appName The app name of the configuration @@ -127,21 +168,62 @@ public class KerberosTestUtil { */ private final static List brokenLanguagesWithMiniKdc = Arrays.asList( - new Locale("th").getLanguage(), - new Locale("ja").getLanguage(), + new Locale("th").getLanguage(), + new Locale("ja").getLanguage(), new Locale("hi").getLanguage() - ); - /** - *returns the currently set locale, and overrides it with {@link Locale#US} if it's - * currently something MiniKdc can not handle - * - * @see Locale#setDefault - */ - public static final Locale overrideLocaleIfNotSpportedByMiniKdc() { - Locale old = Locale.getDefault(); - if (brokenLanguagesWithMiniKdc.contains(Locale.getDefault().getLanguage())) { - Locale.setDefault(Locale.US); + ); + + public static class Builder { + private File kdcWorkDir; + private String clientPrincipal; + private File clientKeytab; + private String serverPrincipal; + private File serverKeytab; + private String appName; + private Locale savedLocale; + + public Builder() { + savedLocale = Locale.getDefault(); + } + + public Builder withKdc(File kdcWorkDir) { + this.kdcWorkDir = kdcWorkDir; + return this; + } + + public Builder withJaasConfiguration(String clientPrincipal, File clientKeytab, + String serverPrincipal, File serverKeytab) { + Preconditions.checkNotNull(clientPrincipal); + Preconditions.checkNotNull(clientKeytab); + this.clientPrincipal = clientPrincipal; + this.clientKeytab = clientKeytab; + this.serverPrincipal = serverPrincipal; + this.serverKeytab = serverKeytab; + this.appName = null; + return this; + } + + public Builder withJaasConfiguration(String principal, File keytab, String appName) { + Preconditions.checkNotNull(principal); + Preconditions.checkNotNull(keytab); + this.clientPrincipal = principal; + this.clientKeytab = keytab; + this.serverPrincipal = null; + this.serverKeytab = null; + this.appName = appName; + return this; + } + + public KerberosTestServices build() throws Exception { + final MiniKdc kdc = kdcWorkDir != null ? getKdc(kdcWorkDir) : null; + final Configuration oldConfig = clientPrincipal != null ? Configuration.getConfiguration() : null; + JaasConfiguration jaasConfiguration = null; + if (clientPrincipal != null) { + jaasConfiguration = (appName == null) ? + new JaasConfiguration(clientPrincipal, clientKeytab, serverPrincipal, serverKeytab) : + new JaasConfiguration(clientPrincipal, clientKeytab, appName); + } + return new KerberosTestServices(kdc, jaasConfiguration, oldConfig, savedLocale); } - return old; } } diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java index 95dccab2a71..8e1be10959a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionTest.java @@ -118,6 +118,7 @@ public class LeaderElectionTest extends SolrTestCaseJ4 { if (!zkClient.isClosed()) { zkClient.close(); } + zkStateReader.close(); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java index 51ad523aca5..08845767af9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.ZooDefs; @@ -77,6 +78,7 @@ public class OutOfBoxZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { zkClient.makePath("/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); zkClient.create("/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); zkClient.makePath("/unprotectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.create(SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); zkClient.close(); log.info("####SETUP_END " + getTestName()); @@ -93,7 +95,9 @@ public class OutOfBoxZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { public void testOutOfBoxSolrZkClient() throws Exception { SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true); + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, + true, true, true, true, true, + true, true, true, true, true); } finally { zkClient.close(); } @@ -110,6 +114,7 @@ public class OutOfBoxZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { assertTrue(verifiedList.contains("/solr/unprotectedMakePathNode")); assertTrue(verifiedList.contains("/solr/protectedMakePathNode")); assertTrue(verifiedList.contains("/solr/protectedCreateNode")); + assertTrue(verifiedList.contains("/solr" + SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH)); } finally { zkClient.close(); } diff --git a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java index b87ab1be261..56c0df90d3b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java @@ -18,18 +18,15 @@ package org.apache.solr.cloud; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.StringUtils; -import org.apache.solr.common.cloud.DefaultZkACLProvider; import org.apache.solr.common.cloud.DefaultZkCredentialsProvider; +import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider; import org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider; import org.apache.solr.common.cloud.ZkACLProvider; import org.apache.solr.common.cloud.ZkCredentialsProvider; import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -40,7 +37,6 @@ import java.io.File; import java.io.UnsupportedEncodingException; import java.lang.invoke.MethodHandles; import java.nio.charset.Charset; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -88,6 +84,7 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { "readonlyACLUsername", "readonlyACLPassword").getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); zkClient.create("/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); zkClient.makePath("/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.create(SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); zkClient.close(); zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders(null, null, @@ -114,7 +111,9 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders(null, null, null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false); + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, + false, false, false, false, false, + false, false, false, false, false); } finally { zkClient.close(); } @@ -125,7 +124,9 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("connectAndAllACLUsername", "connectAndAllACLPasswordWrong", null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false); + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, + false, false, false, false, false, + false, false, false, false, false); } finally { zkClient.close(); } @@ -136,7 +137,9 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("connectAndAllACLUsername", "connectAndAllACLPassword", null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true); + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, + true, true, true, true, true, + true, true, true, true, true); } finally { zkClient.close(); } @@ -147,7 +150,9 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders("readonlyACLUsername", "readonlyACLPassword", null, null).getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, false, false, false); + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, + true, true, false, false, false, + false, false, false, false, false); } finally { zkClient.close(); } @@ -159,7 +164,9 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { SolrZkClient zkClient = new SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false); + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, + false, false, false, false, false, + false, false, false, false, false); } finally { zkClient.close(); } @@ -171,7 +178,9 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { SolrZkClient zkClient = new SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, false, false, false, false, false); + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, + false, false, false, false, false, + false, false, false, false, false); } finally { zkClient.close(); } @@ -183,7 +192,9 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { SolrZkClient zkClient = new SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true); + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, + true, true, true, true, true, + true, true, true, true, true); } finally { zkClient.close(); } @@ -195,7 +206,9 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { SolrZkClient zkClient = new SolrZkClientUsingVMParamsProvidersButWithDifferentVMParamsNames(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, false, false, false); + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, + true, true, false, false, false, + false, false, false, false, false); } finally { zkClient.close(); } @@ -240,28 +253,18 @@ public class OverriddenZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { @Override public ZkACLProvider createZkACLProvider() { - return new DefaultZkACLProvider() { + return new VMParamsAllAndReadonlyDigestZkACLProvider() { @Override - protected List createGlobalACLsToAdd() { - try { - List result = new ArrayList(); - - if (!StringUtils.isEmpty(digestUsername) && !StringUtils.isEmpty(digestPassword)) { - result.add(new ACL(ZooDefs.Perms.ALL, new Id("digest", DigestAuthenticationProvider.generateDigest(digestUsername + ":" + digestPassword)))); - } - - if (!StringUtils.isEmpty(digestReadonlyUsername) && !StringUtils.isEmpty(digestReadonlyPassword)) { - result.add(new ACL(ZooDefs.Perms.READ, new Id("digest", DigestAuthenticationProvider.generateDigest(digestReadonlyUsername + ":" + digestReadonlyPassword)))); - } - - if (result.isEmpty()) { - result = ZooDefs.Ids.OPEN_ACL_UNSAFE; - } - - return result; - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } + protected List createNonSecurityACLsToAdd() { + return createACLsToAdd(true, digestUsername, digestPassword, digestReadonlyUsername, digestReadonlyPassword); + } + + /** + * @return Set of ACLs to return security-related znodes + */ + @Override + protected List createSecurityACLsToAdd() { + return createACLsToAdd(false, digestUsername, digestPassword, digestReadonlyUsername, digestReadonlyPassword); } }; } diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java index 8c9daad6c04..9166a431af2 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java @@ -130,6 +130,7 @@ public class OverseerTest extends SolrTestCaseJ4 { } } deleteNode(ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName); + zkStateReader.close(); zkClient.close(); } diff --git a/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java b/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java index 9381c031971..16b67a34a21 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java @@ -20,15 +20,12 @@ import java.io.File; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.nio.charset.Charset; -import java.util.Locale; -import javax.security.auth.login.Configuration; - -import org.apache.hadoop.minikdc.MiniKdc; import org.apache.lucene.util.Constants; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.DefaultZkACLProvider; import org.apache.solr.common.cloud.SaslZkACLProvider; +import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkACLProvider; import org.apache.solr.util.BadZookeeperThreadsFilter; @@ -50,8 +47,6 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 { private static final Charset DATA_ENCODING = Charset.forName("UTF-8"); - protected Locale savedLocale = null; - protected ZkTestServer zkServer; @BeforeClass @@ -71,7 +66,6 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 { @Override public void setUp() throws Exception { super.setUp(); - savedLocale = KerberosTestUtil.overrideLocaleIfNotSpportedByMiniKdc(); log.info("####SETUP_START " + getTestName()); createTempDir(); @@ -99,6 +93,7 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 { try { zkClient.create("/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); zkClient.makePath("/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.create(SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); } finally { zkClient.close(); } @@ -115,7 +110,6 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 { @Override public void tearDown() throws Exception { zkServer.shutdown(); - Locale.setDefault(savedLocale); super.tearDown(); } @@ -124,7 +118,9 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 { // Test with Sasl enabled SolrZkClient zkClient = new SolrZkClientWithACLs(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, true, true, true); + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, + true, true, true, true, true, + true, true, true, true, true); } finally { zkClient.close(); } @@ -134,7 +130,9 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 { System.setProperty("zookeeper.sasl.client", "false"); zkClient = new SolrZkClientNoACLs(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, true, true, false, false, false); + VMParamsZkACLAndCredentialsProvidersTest.doTest(zkClient, + true, true, false, false, false, + false, false, false, false, false); } finally { zkClient.close(); System.clearProperty("zookeeper.sasl.client"); @@ -176,8 +174,7 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 { */ public static class SaslZkTestServer extends ZkTestServer { private String kdcDir; - private MiniKdc kdc; - private Configuration conf; + private KerberosTestServices kerberosTestServices; public SaslZkTestServer(String zkDir, String kdcDir) { super(zkDir); @@ -187,13 +184,11 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 { public SaslZkTestServer(String zkDir, int port, String kdcDir) { super(zkDir, port); this.kdcDir = kdcDir; - conf = Configuration.getConfiguration(); } @Override public void run() throws InterruptedException { try { - kdc = KerberosTestUtil.getKdc(new File(kdcDir)); // Don't require that credentials match the entire principal string, e.g. // can match "solr" rather than "solr/host@DOMAIN" System.setProperty("zookeeper.kerberos.removeRealmFromPrincipal", "true"); @@ -202,12 +197,13 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 { String zkClientPrincipal = "solr"; String zkServerPrincipal = "zookeeper/127.0.0.1"; - kdc.start(); - // Create ZK client and server principals and load them into the Configuration - kdc.createPrincipal(keytabFile, zkClientPrincipal, zkServerPrincipal); - KerberosTestUtil.JaasConfiguration jaas = new KerberosTestUtil.JaasConfiguration( - zkClientPrincipal, keytabFile, zkServerPrincipal, keytabFile); - Configuration.setConfiguration(jaas); + kerberosTestServices = KerberosTestServices.builder() + .withKdc(new File(kdcDir)) + .withJaasConfiguration(zkClientPrincipal, keytabFile, zkServerPrincipal, keytabFile) + .build(); + kerberosTestServices.start(); + + kerberosTestServices.getKdc().createPrincipal(keytabFile, zkClientPrincipal, zkServerPrincipal); } catch (Exception ex) { throw new RuntimeException(ex); } @@ -220,8 +216,7 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 { System.clearProperty("zookeeper.authProvider.1"); System.clearProperty("zookeeper.kerberos.removeRealmFromPrincipal"); System.clearProperty("zookeeper.kerberos.removeHostFromPrincipal"); - Configuration.setConfiguration(conf); - kdc.stop(); + kerberosTestServices.stop(); } } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java b/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java index 08db69266dd..6b562a1815b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java @@ -235,21 +235,23 @@ public class TestAuthenticationFramework extends LuceneTestCase { public void init(Map pluginConfig) {} @Override - public void doAuthenticate(ServletRequest request, ServletResponse response, FilterChain filterChain) + public boolean doAuthenticate(ServletRequest request, ServletResponse response, FilterChain filterChain) throws Exception { if (expectedUsername == null) { filterChain.doFilter(request, response); - return; + return true; } HttpServletRequest httpRequest = (HttpServletRequest)request; String username = httpRequest.getHeader("username"); String password = httpRequest.getHeader("password"); log.info("Username: "+username+", password: "+password); - if(MockAuthenticationPlugin.expectedUsername.equals(username) && MockAuthenticationPlugin.expectedPassword.equals(password)) + if(MockAuthenticationPlugin.expectedUsername.equals(username) && MockAuthenticationPlugin.expectedPassword.equals(password)) { filterChain.doFilter(request, response); - else { + return true; + } else { ((HttpServletResponse)response).sendError(401, "Unauthorized request"); + return false; } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java index 8553697d6c2..6a15e682bd0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java @@ -51,6 +51,7 @@ import org.apache.lucene.util.TestUtil; import org.apache.commons.lang.StringUtils; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; /** @@ -104,11 +105,17 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase { assertEquals(0, CLOUD_CLIENT.add(sdoc("id", "46", "val_i", "3", "ssto", "X", "subject", "ggg")).getStatus()); assertEquals(0, CLOUD_CLIENT.commit().getStatus());; - // uncommitted doc in transaction log + } + + @Before + private void addUncommittedDoc99() throws Exception { + // uncommitted doc in transaction log at start of every test + // Even if an RTG causes ulog to re-open realtime searcher, next test method + // will get another copy of doc 99 in the ulog assertEquals(0, CLOUD_CLIENT.add(sdoc("id", "99", "val_i", "1", "ssto", "X", "subject", "uncommitted")).getStatus()); } - + @AfterClass private static void afterClass() throws Exception { CLOUD_CLIENT.close(); CLOUD_CLIENT = null; @@ -170,13 +177,12 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase { assertEquals(""+doc, 10L, doc.getFieldValue("val2_ss")); } - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-9286") public void testMultiValuedRTG() throws Exception { SolrDocument doc = null; // check same results as testMultiValued via RTG (committed doc) doc = getRandClient(random()).getById("42", params("fl","val_ss:val_i, val2_ss:10, subject")); - assertEquals(""+doc, 2, doc.size()); + assertEquals(""+doc, 3, doc.size()); assertEquals(""+doc, 1, doc.getFieldValue("val_ss")); assertEquals(""+doc, 10L, doc.getFieldValue("val2_ss")); assertEquals(""+doc, "aaa", doc.getFieldValue("subject")); @@ -218,6 +224,21 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase { } } } + + public void testFilterAndOneRealFieldRTG() throws Exception { + SolrParams params = params("fl","id,val_i", + "fq","{!field f='subject' v=$my_var}", + "my_var","uncommitted"); + SolrDocumentList docs = getRandClient(random()).getById(Arrays.asList("42","99"), params); + final String msg = params + " => " + docs; + assertEquals(msg, 1, docs.size()); + assertEquals(msg, 1, docs.getNumFound()); + + SolrDocument doc = docs.get(0); + assertEquals(msg, 2, doc.size()); + assertEquals(msg, "99", doc.getFieldValue("id")); + assertEquals(msg, 1, doc.getFieldValue("val_i")); + } public void testScoreAndAllRealFields() throws Exception { for (String fl : TestPseudoReturnFields.SCORE_AND_REAL_FIELDS) { @@ -304,7 +325,6 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase { } } - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-9286") public void testFunctionsRTG() throws Exception { // if we use RTG (committed or otherwise) functions should behave the same for (String id : Arrays.asList("42","99")) { @@ -334,7 +354,6 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase { } } - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-9286") public void testFunctionsAndExplicitRTG() throws Exception { // shouldn't matter if we use RTG (committed or otherwise) for (String id : Arrays.asList("42","99")) { @@ -382,7 +401,6 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase { } } - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-9286") public void testFunctionsAndScoreRTG() throws Exception { // if we use RTG (committed or otherwise) score should be ignored @@ -578,40 +596,35 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase { } } - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-9289") public void testDocIdAugmenterRTG() throws Exception { - // NOTE: once this test is fixed to pass, testAugmentersRTG should also be updated to test [docid] - - // TODO: in single node, [docid] is silently ignored for uncommited docs (see SOLR-9288) ... - // here we see even more confusing: [docid] is silently ignored for both committed & uncommited docs - - // behavior shouldn't matter if we are committed or uncommitted + // for an uncommitted doc, we should get -1 for (String id : Arrays.asList("42","99")) { SolrDocument doc = getRandClient(random()).getById(id, params("fl","[docid]")); String msg = id + ": fl=[docid] => " + doc; assertEquals(msg, 1, doc.size()); assertTrue(msg, doc.getFieldValue("[docid]") instanceof Integer); + assertTrue(msg, -1 <= ((Integer)doc.getFieldValue("[docid]")).intValue()); } } - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-9286") public void testAugmentersRTG() throws Exception { // behavior shouldn't matter if we are committed or uncommitted for (String id : Arrays.asList("42","99")) { - // NOTE: once testDocIdAugmenterRTG can pass, [docid] should be tested here as well. - for (SolrParams p : Arrays.asList(params("fl","[shard],[explain],x_alias:[value v=10 t=int]"), - params("fl","[shard]","fl","[explain],x_alias:[value v=10 t=int]"), - params("fl","[shard]","fl","[explain]","fl","x_alias:[value v=10 t=int]"))) { + for (SolrParams p : Arrays.asList + (params("fl","[docid],[shard],[explain],x_alias:[value v=10 t=int]"), + params("fl","[docid],[shard]","fl","[explain],x_alias:[value v=10 t=int]"), + params("fl","[docid]","fl","[shard]","fl","[explain]","fl","x_alias:[value v=10 t=int]"))) { SolrDocument doc = getRandClient(random()).getById(id, p); String msg = id + ": " + p + " => " + doc; - assertEquals(msg, 2, doc.size()); - // assertTrue(msg, doc.getFieldValue("[docid]") instanceof Integer); // TODO + assertEquals(msg, 3, doc.size()); assertTrue(msg, doc.getFieldValue("[shard]") instanceof String); // RTG: [explain] should be ignored assertTrue(msg, doc.getFieldValue("x_alias") instanceof Integer); assertEquals(msg, 10, doc.getFieldValue("x_alias")); + assertTrue(msg, doc.getFieldValue("[docid]") instanceof Integer); + assertTrue(msg, -1 <= ((Integer)doc.getFieldValue("[docid]")).intValue()); } } } @@ -635,23 +648,22 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase { } } - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-9286") public void testAugmentersAndExplicitRTG() throws Exception { // behavior shouldn't matter if we are committed or uncommitted for (String id : Arrays.asList("42","99")) { - // NOTE: once testDocIdAugmenterRTG can pass, [docid] should be tested here as well. - for (SolrParams p : Arrays.asList(params("fl","id,[explain],x_alias:[value v=10 t=int]"), - params("fl","id","fl","[explain],x_alias:[value v=10 t=int]"), - params("fl","id","fl","[explain]","fl","x_alias:[value v=10 t=int]"))) { + for (SolrParams p : Arrays.asList(params("fl","id,[docid],[explain],x_alias:[value v=10 t=int]"), + params("fl","id,[docid]","fl","[explain],x_alias:[value v=10 t=int]"), + params("fl","id","fl","[docid]","fl","[explain]","fl","x_alias:[value v=10 t=int]"))) { SolrDocument doc = getRandClient(random()).getById(id, p); String msg = id + ": " + p + " => " + doc; - assertEquals(msg, 2, doc.size()); + assertEquals(msg, 3, doc.size()); assertTrue(msg, doc.getFieldValue("id") instanceof String); - // assertTrue(msg, doc.getFieldValue("[docid]") instanceof Integer); // TODO // RTG: [explain] should be missing (ignored) assertTrue(msg, doc.getFieldValue("x_alias") instanceof Integer); assertEquals(msg, 10, doc.getFieldValue("x_alias")); + assertTrue(msg, doc.getFieldValue("[docid]") instanceof Integer); + assertTrue(msg, -1 <= ((Integer)doc.getFieldValue("[docid]")).intValue()); } } } @@ -688,32 +700,29 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase { } } - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-9286") public void testAugmentersAndScoreRTG() throws Exception { // if we use RTG (committed or otherwise) score should be ignored for (String id : Arrays.asList("42","99")) { - // NOTE: once testDocIdAugmenterRTG can pass, [docid] should be tested here as well. SolrDocument doc = getRandClient(random()).getById(id, params("fl","x_alias:[value v=10 t=int],score")); String msg = id + " => " + doc; assertEquals(msg, 1, doc.size()); - // assertTrue(msg, doc.getFieldValue("[docid]") instanceof Integer); // TODO assertTrue(msg, doc.getFieldValue("x_alias") instanceof Integer); assertEquals(msg, 10, doc.getFieldValue("x_alias")); - for (SolrParams p : Arrays.asList(params("fl","x_alias:[value v=10 t=int],[explain],score"), - params("fl","x_alias:[value v=10 t=int],[explain]","fl","score"), - params("fl","x_alias:[value v=10 t=int]","fl","[explain]","fl","score"))) { + for (SolrParams p : Arrays.asList(params("fl","d_alias:[docid],x_alias:[value v=10 t=int],[explain],score"), + params("fl","d_alias:[docid],x_alias:[value v=10 t=int],[explain]","fl","score"), + params("fl","d_alias:[docid]","fl","x_alias:[value v=10 t=int]","fl","[explain]","fl","score"))) { doc = getRandClient(random()).getById(id, p); msg = id + ": " + p + " => " + doc; - assertEquals(msg, 1, doc.size()); - assertTrue(msg, doc.getFieldValue("id") instanceof String); - // assertTrue(msg, doc.getFieldValue("[docid]") instanceof Integer); // TODO + assertEquals(msg, 2, doc.size()); assertTrue(msg, doc.getFieldValue("x_alias") instanceof Integer); assertEquals(msg, 10, doc.getFieldValue("x_alias")); // RTG: [explain] and score should be missing (ignored) + assertTrue(msg, doc.getFieldValue("d_alias") instanceof Integer); + assertTrue(msg, -1 <= ((Integer)doc.getFieldValue("d_alias")).intValue()); } } } @@ -758,8 +767,7 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase { // NOTE: 'ssto' is the missing one final List fl = Arrays.asList - // NOTE: once testDocIdAugmenterRTG can pass, [docid] should be tested here as well. - ("id","[explain]","score","val_*","subj*"); + ("id","[docid]","[explain]","score","val_*","subj*"); final int iters = atLeast(random, 10); for (int i = 0; i< iters; i++) { @@ -778,12 +786,13 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase { SolrDocument doc = getRandClient(random()).getById(id, params); String msg = id + ": " + params + " => " + doc; - assertEquals(msg, 3, doc.size()); + assertEquals(msg, 4, doc.size()); assertTrue(msg, doc.getFieldValue("id") instanceof String); - // assertTrue(msg, doc.getFieldValue("[docid]") instanceof Integer); // TODO assertTrue(msg, doc.getFieldValue("val_i") instanceof Integer); assertEquals(msg, 1, doc.getFieldValue("val_i")); assertTrue(msg, doc.getFieldValue("subject") instanceof String); + assertTrue(msg, doc.getFieldValue("[docid]") instanceof Integer); + assertTrue(msg, -1 <= ((Integer)doc.getFieldValue("[docid]")).intValue()); // RTG: [explain] and score should be missing (ignored) } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterKerberos.java b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterKerberos.java index 37439b07b8f..d644967481f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterKerberos.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterKerberos.java @@ -16,16 +16,13 @@ */ package org.apache.solr.cloud; -import javax.security.auth.login.Configuration; import java.io.File; import java.nio.charset.StandardCharsets; -import java.util.Locale; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; import org.apache.commons.io.FileUtils; -import org.apache.hadoop.minikdc.MiniKdc; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks; import org.apache.solr.util.BadZookeeperThreadsFilter; @@ -52,17 +49,14 @@ import org.junit.rules.TestRule; @SuppressSysoutChecks(bugUrl = "Solr logs to JUL") public class TestMiniSolrCloudClusterKerberos extends TestMiniSolrCloudCluster { - private final Configuration originalConfig = Configuration.getConfiguration(); - public TestMiniSolrCloudClusterKerberos () { NUM_SERVERS = 5; NUM_SHARDS = 2; REPLICATION_FACTOR = 2; } - private MiniKdc kdc; + private KerberosTestServices kerberosTestServices; - private Locale savedLocale; // in case locale is broken and we need to fill in a working locale @Rule public TestRule solrTestRules = RuleChain .outerRule(new SystemPropertiesRestoreRule()); @@ -74,20 +68,22 @@ public class TestMiniSolrCloudClusterKerberos extends TestMiniSolrCloudCluster { @Override public void setUp() throws Exception { - savedLocale = KerberosTestUtil.overrideLocaleIfNotSpportedByMiniKdc(); super.setUp(); setupMiniKdc(); } private void setupMiniKdc() throws Exception { String kdcDir = createTempDir()+File.separator+"minikdc"; - kdc = KerberosTestUtil.getKdc(new File(kdcDir)); File keytabFile = new File(kdcDir, "keytabs"); String principal = "HTTP/127.0.0.1"; String zkServerPrincipal = "zookeeper/127.0.0.1"; + KerberosTestServices kerberosTestServices = KerberosTestServices.builder() + .withKdc(new File(kdcDir)) + .withJaasConfiguration(principal, keytabFile, zkServerPrincipal, keytabFile) + .build(); - kdc.start(); - kdc.createPrincipal(keytabFile, principal, zkServerPrincipal); + kerberosTestServices.start(); + kerberosTestServices.getKdc().createPrincipal(keytabFile, principal, zkServerPrincipal); String jaas = "Client {\n" + " com.sun.security.auth.module.Krb5LoginModule required\n" @@ -109,10 +105,7 @@ public class TestMiniSolrCloudClusterKerberos extends TestMiniSolrCloudCluster { + " debug=true\n" + " principal=\""+zkServerPrincipal+"\";\n" + "};\n"; - - Configuration conf = new KerberosTestUtil.JaasConfiguration(principal, keytabFile, zkServerPrincipal, keytabFile); - javax.security.auth.login.Configuration.setConfiguration(conf); - + String jaasFilePath = kdcDir+File.separator + "jaas-client.conf"; FileUtils.write(new File(jaasFilePath), jaas, StandardCharsets.UTF_8); System.setProperty("java.security.auth.login.config", jaasFilePath); @@ -156,11 +149,7 @@ public class TestMiniSolrCloudClusterKerberos extends TestMiniSolrCloudCluster { System.clearProperty("kerberos.principal"); System.clearProperty("kerberos.keytab"); System.clearProperty("authenticationPlugin"); - Configuration.setConfiguration(this.originalConfig); - if (kdc != null) { - kdc.stop(); - } - Locale.setDefault(savedLocale); + kerberosTestServices.stop(); super.tearDown(); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java index 8fc61c7be0e..2e54679b085 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java @@ -26,11 +26,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.Random; +import java.util.Set; +import java.util.TreeSet; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; @@ -47,7 +48,10 @@ import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; -import org.apache.solr.response.transform.DocTransformer; // jdocs +import org.apache.solr.response.transform.DocTransformer; // jdoc +import org.apache.solr.response.transform.RawValueTransformerFactory; // jdoc +import org.apache.solr.response.transform.TransformerFactory; + import org.apache.solr.util.RandomizeSSL; import org.apache.lucene.util.TestUtil; @@ -78,58 +82,54 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { /** * Types of things we will randomly ask for in fl param, and validate in response docs. * - * This list starts out with the things we know concretely should work for any type of request, - * {@link #createMiniSolrCloudCluster} will add too it with additional validators that are expected - * to work dependingon hte random cluster creation - * * @see #addRandomFlValidators */ - private static final List FL_VALIDATORS = new ArrayList<> - // TODO: SOLR-9314: once all the known bugs are fixed, and this list can be constant - // regardless of single/multi node, change this to Collections.unmodifiableList - // (and adjust jdocs accordingly) + private static final List FL_VALIDATORS = Collections.unmodifiableList (Arrays.asList( - // TODO: SOLR-9314: add more of these for other various transformers - // new GlobValidator("*"), new GlobValidator("*_i"), new GlobValidator("*_s"), new GlobValidator("a*"), + new DocIdValidator(), + new DocIdValidator("my_docid_alias"), + new ShardValidator(), + new ShardValidator("my_shard_alias"), + new ValueAugmenterValidator(42), + new ValueAugmenterValidator(1976, "val_alias"), + // + new RenameFieldValueValidator("id", "my_id_alias"), new SimpleFieldValueValidator("aaa_i"), + new RenameFieldValueValidator("bbb_i", "my_int_field_alias"), new SimpleFieldValueValidator("ccc_s"), + new RenameFieldValueValidator("ddd_s", "my_str_field_alias"), + // + // SOLR-9376: RawValueTransformerFactory doesn't work in cloud mode + // + // new RawFieldValueValidator("json", "eee_s", "my_json_field_alias"), + // new RawFieldValueValidator("json", "fff_s"), + // new RawFieldValueValidator("xml", "ggg_s", "my_xml_field_alias"), + // new RawFieldValueValidator("xml", "hhh_s"), + // new NotIncludedValidator("bogus_unused_field_ss"), new NotIncludedValidator("bogus_alias","bogus_alias:other_bogus_field_i"), - new NotIncludedValidator("explain_alias","explain_alias:[explain]"), - new NotIncludedValidator("score"))); + new NotIncludedValidator("bogus_raw_alias","bogus_raw_alias:[xml f=bogus_raw_field_ss]"), + // + new FunctionValidator("aaa_i"), // fq field + new FunctionValidator("aaa_i", "func_aaa_alias"), + new GeoTransformerValidator("geo_1_srpt"), + new GeoTransformerValidator("geo_2_srpt","my_geo_alias"), + new ExplainValidator(), + new ExplainValidator("explain_alias"), + new SubQueryValidator(), + new NotIncludedValidator("score"), + new NotIncludedValidator("score","score_alias:score"))); @BeforeClass private static void createMiniSolrCloudCluster() throws Exception { - // Due to known bugs with some transformers in either multi vs single node, we want - // to test both possible cases explicitly and modify the List of FL_VALIDATORS we use accordingly: - // - 50% runs use single node/shard a FL_VALIDATORS with all validators known to work on single node - // - 50% runs use multi node/shard with FL_VALIDATORS only containing stuff that works in cloud + // 50% runs use single node/shard a FL_VALIDATORS with all validators known to work on single node + // 50% runs use multi node/shard with FL_VALIDATORS only containing stuff that works in cloud final boolean singleCoreMode = random().nextBoolean(); - if (singleCoreMode) { - // these don't work in distrib cloud mode due to SOLR-9286 - FL_VALIDATORS.addAll(Arrays.asList - (new FunctionValidator("aaa_i"), // fq field - new FunctionValidator("aaa_i", "func_aaa_alias"), - new RenameFieldValueValidator("id", "my_id_alias"), - new RenameFieldValueValidator("bbb_i", "my_int_field_alias"), - new RenameFieldValueValidator("ddd_s", "my_str_field_alias"))); - // SOLR-9289... - FL_VALIDATORS.add(new DocIdValidator()); - FL_VALIDATORS.add(new DocIdValidator("my_docid_alias")); - } else { - // No-Op - // No known transformers that only work in distrib cloud but fail in singleCoreMode - - } - // TODO: SOLR-9314: programatically compare FL_VALIDATORS with all known transformers. - // (ala QueryEqualityTest) can't be done until we eliminate the need for "singleCodeMode" - // conditional logic (might still want 'singleCoreMode' on the MiniSolrCloudCluster side, - // but shouldn't have conditional FlValidators // (asuming multi core multi replicas shouldn't matter (assuming multi node) ... final int repFactor = singleCoreMode ? 1 : (usually() ? 1 : 2); @@ -168,7 +168,50 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { } CLIENTS = null; } - + + /** + * Tests thta all TransformerFactories that are implicitly provided by Solr are tested in this class + * + * @see FlValidator#getDefaultTransformerFactoryName + * @see #FL_VALIDATORS + * @see TransformerFactory#defaultFactories + */ + public void testCoverage() throws Exception { + final Set implicit = new LinkedHashSet<>(); + for (String t : TransformerFactory.defaultFactories.keySet()) { + implicit.add(t); + } + + final Set covered = new LinkedHashSet<>(); + for (FlValidator v : FL_VALIDATORS) { + String t = v.getDefaultTransformerFactoryName(); + if (null != t) { + covered.add(t); + } + } + + // items should only be added to this list if it's known that they do not work with RTG + // and a specific Jira for fixing this is listed as a comment + final List knownBugs = Arrays.asList + ( "xml","json", // SOLR-9376 + "child" // way to complicatd to vet with this test, see SOLR-9379 instead + ); + + for (String buggy : knownBugs) { + assertFalse(buggy + " is listed as a being a known bug, " + + "but it exists in the set of 'covered' TransformerFactories", + covered.contains(buggy)); + assertTrue(buggy + " is listed as a known bug, " + + "but it does not even exist in the set of 'implicit' TransformerFactories", + implicit.remove(buggy)); + } + + implicit.removeAll(covered); + assertEquals("Some implicit TransformerFactories are not yet tested by this class: " + implicit, + 0, implicit.size()); + } + + public void testRandomizedUpdatesAndRTGs() throws Exception { final int maxNumDocs = atLeast(100); @@ -274,14 +317,24 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { */ private SolrInputDocument addRandomDocument(final int docId) throws IOException, SolrServerException { final SolrClient client = getRandClient(random()); - + final SolrInputDocument doc = sdoc("id", "" + docId, "aaa_i", random().nextInt(), "bbb_i", random().nextInt(), // "ccc_s", TestUtil.randomSimpleString(random()), "ddd_s", TestUtil.randomSimpleString(random()), + "eee_s", TestUtil.randomSimpleString(random()), + "fff_s", TestUtil.randomSimpleString(random()), + "ggg_s", TestUtil.randomSimpleString(random()), + "hhh_s", TestUtil.randomSimpleString(random()), // + "geo_1_srpt", GeoTransformerValidator.getValueForIndexing(random()), + "geo_2_srpt", GeoTransformerValidator.getValueForIndexing(random()), + // for testing subqueries + "next_2_ids_ss", String.valueOf(docId + 1), + "next_2_ids_ss", String.valueOf(docId + 2), + // for testing prefix globbing "axx_i", random().nextInt(), "ayy_i", random().nextInt(), "azz_s", TestUtil.randomSimpleString(random())); @@ -301,21 +354,17 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { // NOTE: not using SolrClient.getById or getByIds because we want to force choice of "id" vs "ids" params final ModifiableSolrParams params = params("qt","/get"); - // TODO: fq testing blocked by SOLR-9308 - // - // // random fq -- nothing fancy, secondary concern for our test - final Integer FQ_MAX = null; // TODO: replace this... - // final Integer FQ_MAX = usually() ? null : random().nextInt(); // ... with this - // if (null != FQ_MAX) { - // params.add("fq", "aaa_i:[* TO " + FQ_MAX + "]"); - // } - // TODO: END + // random fq -- nothing fancy, secondary concern for our test + final Integer FQ_MAX = usually() ? null : random().nextInt(); + if (null != FQ_MAX) { + params.add("fq", "aaa_i:[* TO " + FQ_MAX + "]"); + } - final Set validators = new HashSet<>(); + final Set validators = new LinkedHashSet<>(); validators.add(ID_VALIDATOR); // always include id so we can be confident which doc we're looking at addRandomFlValidators(random(), validators); - FlValidator.addFlParams(validators, params); - + FlValidator.addParams(validators, params); + final List idsToRequest = new ArrayList<>(docIds.length); final List docsToExpect = new ArrayList<>(docIds.length); for (int docId : docIds) { @@ -366,17 +415,17 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { // NOTE: RTG makes no garuntees about the order docs will be returned in when multi requested for (SolrDocument actual : docs) { try { - int actualId = Integer.parseInt(actual.getFirstValue("id").toString()); + int actualId = assertParseInt("id", actual.getFirstValue("id")); final SolrInputDocument expected = knownDocs[actualId]; assertNotNull("expected null doc but RTG returned: " + actual, expected); - Set expectedFieldNames = new HashSet<>(); + Set expectedFieldNames = new TreeSet<>(); for (FlValidator v : validators) { expectedFieldNames.addAll(v.assertRTGResults(validators, expected, actual)); } // ensure only expected field names are in the actual document - Set actualFieldNames = new HashSet<>(actual.getFieldNames()); - assertEquals("More actual fields then expected", expectedFieldNames, actualFieldNames); + Set actualFieldNames = new TreeSet<>(actual.getFieldNames()); + assertEquals("Actual field names returned differs from expected", expectedFieldNames, actualFieldNames); } catch (AssertionError ae) { throw new AssertionError(params + " => " + actual + ": " + ae.getMessage(), ae); } @@ -430,10 +479,14 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { */ private interface FlValidator { - /** Given a list of FlValidators, adds one or more fl params that corrispond to the entire set */ - public static void addFlParams(final Collection validators, final ModifiableSolrParams params) { + /** + * Given a list of FlValidators, adds one or more fl params that corrispond to the entire set, + * as well as any other special case top level params required by the validators. + */ + public static void addParams(final Collection validators, final ModifiableSolrParams params) { final List fls = new ArrayList<>(validators.size()); for (FlValidator v : validators) { + params.add(v.getExtraRequestParams()); fls.add(v.getFlParam()); } params.add(buildCommaSepParams(random(), "fl", fls)); @@ -453,6 +506,21 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { public default boolean requiresRealtimeSearcherReOpen() { return false; } + + /** + * the name of a transformer listed in {@link TransformerFactory#defaultFactories} that this validator + * corrisponds to, or null if not applicable. Used for testing coverage of + * Solr's implicitly supported transformers. + * + * Default behavior is to return null + * @see #testCoverage + */ + public default String getDefaultTransformerFactoryName() { return null; } + + /** + * Any special case params that must be added to the request for this validator + */ + public default SolrParams getExtraRequestParams() { return params(); } /** * Must return a non null String that can be used in an fl param -- either by itself, @@ -474,6 +542,14 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { final SolrInputDocument expected, final SolrDocument actual); } + + /** + * Some validators behave in a way that "suppresses" real fields even when they would otherwise match a glob + * @see GlobValidator + */ + private interface SuppressRealFields { + public Set getSuppressedFields(); + } private abstract static class FieldValueValidator implements FlValidator { protected final String expectedFieldName; @@ -499,15 +575,45 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { public String getFlParam() { return expectedFieldName; } } - private static class RenameFieldValueValidator extends FieldValueValidator { - /** @see GlobValidator */ - public String getRealFieldName() { return expectedFieldName; } + private static class RenameFieldValueValidator extends FieldValueValidator implements SuppressRealFields { public RenameFieldValueValidator(final String origFieldName, final String alias) { super(origFieldName, alias); } public String getFlParam() { return actualFieldName + ":" + expectedFieldName; } + public Set getSuppressedFields() { return Collections.singleton(expectedFieldName); } } + /** + * Validator for {@link RawValueTransformerFactory} + * + * This validator is fairly weak, because it doesn't do anything to verify the conditional logic + * in RawValueTransformerFactory realted to the output format -- but that's out of the scope of + * this randomized testing. + * + * What we're primarily concerned with is that the transformer does it's job and puts the string + * in the response, regardless of cloud/RTG/uncommited state of the document. + */ + private static class RawFieldValueValidator extends RenameFieldValueValidator { + final String type; + final String alias; + public RawFieldValueValidator(final String type, final String fieldName, final String alias) { + // transformer is weird, default result key doesn't care what params are used... + super(fieldName, null == alias ? "["+type+"]" : alias); + this.type = type; + this.alias = alias; + } + public RawFieldValueValidator(final String type, final String fieldName) { + this(type, fieldName, null); + } + public String getFlParam() { + return (null == alias ? "" : (alias + ":")) + "[" + type + " f=" + expectedFieldName + "]"; + } + public String getDefaultTransformerFactoryName() { + return type; + } + } + + /** * enforces that a valid [docid] is present in the response, possibly using a * resultKey alias. By default the only validation of docId values is that they are an integer @@ -516,20 +622,23 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { * be greater than or equal to 0 */ private static class DocIdValidator implements FlValidator { + private static final String NAME = "docid"; + private static final String USAGE = "["+NAME+"]"; private final String resultKey; public DocIdValidator(final String resultKey) { this.resultKey = resultKey; } public DocIdValidator() { - this("[docid]"); + this(USAGE); } - public String getFlParam() { return "[docid]".equals(resultKey) ? resultKey : resultKey+":[docid]"; } + public String getDefaultTransformerFactoryName() { return NAME; } + public String getFlParam() { return USAGE.equals(resultKey) ? resultKey : resultKey+":"+USAGE; } public Collection assertRTGResults(final Collection validators, final SolrInputDocument expected, final SolrDocument actual) { final Object value = actual.getFirstValue(resultKey); assertNotNull(getFlParam() + " => no value in actual doc", value); - assertTrue("[docid] must be an Integer: " + value, value instanceof Integer); + assertTrue(USAGE + " must be an Integer: " + value, value instanceof Integer); int minValidDocId = -1; // if it comes from update log for (FlValidator other : validators) { @@ -538,11 +647,70 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { break; } } - assertTrue("[docid] must be >= " + minValidDocId + ": " + value, + assertTrue(USAGE + " must be >= " + minValidDocId + ": " + value, minValidDocId <= ((Integer)value).intValue()); return Collections.singleton(resultKey); } } + + /** Trivial validator of ShardAugmenterFactory */ + private static class ShardValidator implements FlValidator { + private static final String NAME = "shard"; + private static final String USAGE = "["+NAME+"]"; + private final String resultKey; + public ShardValidator(final String resultKey) { + this.resultKey = resultKey; + } + public ShardValidator() { + this(USAGE); + } + public String getDefaultTransformerFactoryName() { return NAME; } + public String getFlParam() { return USAGE.equals(resultKey) ? resultKey : resultKey+":"+USAGE; } + public Collection assertRTGResults(final Collection validators, + final SolrInputDocument expected, + final SolrDocument actual) { + final Object value = actual.getFirstValue(resultKey); + assertNotNull(getFlParam() + " => no value in actual doc", value); + assertTrue(USAGE + " must be an String: " + value, value instanceof String); + + // trivial sanity check + assertFalse(USAGE + " => blank string", value.toString().trim().isEmpty()); + return Collections.singleton(resultKey); + } + } + + /** Trivial validator of ValueAugmenter */ + private static class ValueAugmenterValidator implements FlValidator { + private static final String NAME = "value"; + private static String trans(final int value) { return "[" + NAME + " v=" + value + " t=int]"; } + + private final String resultKey; + private final String fl; + private final Integer expectedVal; + private ValueAugmenterValidator(final String fl, final int expectedVal, final String resultKey) { + this.resultKey = resultKey; + this.expectedVal = expectedVal; + this.fl = fl; + } + public ValueAugmenterValidator(final int expectedVal, final String resultKey) { + this(resultKey + ":" +trans(expectedVal), expectedVal, resultKey); + } + public ValueAugmenterValidator(final int expectedVal) { + // value transformer is weird, default result key doesn't care what params are used... + this(trans(expectedVal), expectedVal, "["+NAME+"]"); + } + public String getDefaultTransformerFactoryName() { return NAME; } + public String getFlParam() { return fl; } + public Collection assertRTGResults(final Collection validators, + final SolrInputDocument expected, + final SolrDocument actual) { + final Object actualVal = actual.getFirstValue(resultKey); + assertNotNull(getFlParam() + " => no value in actual doc", actualVal); + assertEquals(getFlParam(), expectedVal, actualVal); + return Collections.singleton(resultKey); + } + } + /** Trivial validator of a ValueSourceAugmenter */ private static class FunctionValidator implements FlValidator { @@ -577,10 +745,121 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { } } + + /** + * Trivial validator of a SubQueryAugmenter. + * + * This validator ignores 90% of the features/complexity + * of SubQueryAugmenter, and instead just focuses on the basics of: + *

    + *
  • do a subquery for docs where SUBQ_FIELD contains the id of the top level doc
    • + *
  • verify that any subquery match is expected based on indexing pattern
    • + *
+ */ + private static class SubQueryValidator implements FlValidator { + + // HACK to work around SOLR-9396... + // + // we're using "id" (and only "id") in the subquery.q as a workarround limitation in + // "$rows.foo" parsing -- it only works reliably if "foo" is in fl, so we only use "$rows.id", + // which we know is in every request (and is a valid integer) + + public final static String NAME = "subquery"; + public final static String SUBQ_KEY = "subq"; + public final static String SUBQ_FIELD = "next_2_ids_i"; + public String getFlParam() { return SUBQ_KEY+":["+NAME+"]"; } + public Collection assertRTGResults(final Collection validators, + final SolrInputDocument expected, + final SolrDocument actual) { + final int compVal = assertParseInt("expected id", expected.getFieldValue("id")); + + final Object actualVal = actual.getFieldValue(SUBQ_KEY); + assertTrue("Expected a doclist: " + actualVal, + actualVal instanceof SolrDocumentList); + assertTrue("should be at most 2 docs in doc list: " + actualVal, + ((SolrDocumentList) actualVal).getNumFound() <= 2); + + for (SolrDocument subDoc : (SolrDocumentList) actualVal) { + final int subDocIdVal = assertParseInt("subquery id", subDoc.getFirstValue("id")); + assertTrue("subDocId="+subDocIdVal+" not in valid range for id="+compVal+" (expected " + + (compVal-1) + " or " + (compVal-2) + ")", + ((subDocIdVal < compVal) && ((compVal-2) <= subDocIdVal))); + + } + + return Collections.singleton(SUBQ_KEY); + } + public String getDefaultTransformerFactoryName() { return NAME; } + public SolrParams getExtraRequestParams() { + return params(SubQueryValidator.SUBQ_KEY + ".q", + "{!field f=" + SubQueryValidator.SUBQ_FIELD + " v=$row.id}"); + } + } + + /** Trivial validator of a GeoTransformer */ + private static class GeoTransformerValidator implements FlValidator, SuppressRealFields{ + private static final String NAME = "geo"; + /** + * we're not worried about testing the actual geo parsing/formatting of values, + * just that the transformer gets called with the expected field value. + * so have a small set of fixed input values we use when indexing docs, + * and the expected output for each + */ + private static final Map VALUES = new HashMap<>(); + /** + * The set of legal field values this validator is willing to test as a list so we can + * reliably index into it with random ints + */ + private static final List ALLOWED_FIELD_VALUES; + static { + for (int i = -42; i < 66; i+=13) { + VALUES.put("POINT( 42 "+i+" )", "{\"type\":\"Point\",\"coordinates\":[42,"+i+"]}"); + } + ALLOWED_FIELD_VALUES = Collections.unmodifiableList(new ArrayList<>(VALUES.keySet())); + } + /** + * returns a random field value usable when indexing a document that this validator will + * be able to handle. + */ + public static String getValueForIndexing(final Random rand) { + return ALLOWED_FIELD_VALUES.get(rand.nextInt(ALLOWED_FIELD_VALUES.size())); + } + private static String trans(String fieldName) { + return "["+NAME+" f="+fieldName+"]"; + } + protected final String fl; + protected final String resultKey; + protected final String fieldName; + public GeoTransformerValidator(final String fieldName) { + // geo transformer is weird, default result key doesn't care what params are used... + this(trans(fieldName), fieldName, "["+NAME+"]"); + } + public GeoTransformerValidator(final String fieldName, final String resultKey) { + this(resultKey + ":" + trans(fieldName), fieldName, resultKey); + } + private GeoTransformerValidator(final String fl, final String fieldName, final String resultKey) { + this.fl = fl; + this.resultKey = resultKey; + this.fieldName = fieldName; + } + public String getDefaultTransformerFactoryName() { return NAME; } + public String getFlParam() { return fl; } + public Collection assertRTGResults(final Collection validators, + final SolrInputDocument expected, + final SolrDocument actual) { + final Object origVal = expected.getFieldValue(fieldName); + assertTrue(fl + ": orig field value is not supported: " + origVal, VALUES.containsKey(origVal)); + + assertEquals(fl, VALUES.get(origVal), actual.getFirstValue(resultKey)); + return Collections.singleton(resultKey); + } + public Set getSuppressedFields() { return Collections.singleton(fieldName); } + } + /** * Glob based validator. * This class checks that every field in the expected doc exists in the actual doc with the expected - * value -- with special exceptions for fields that are "renamed" with an alias. + * value -- with special exceptions for fields that are "suppressed" (usually via an alias) * * By design, fields that are aliased are "moved" unless the original field name was explicitly included * in the fl, globs don't count. @@ -592,7 +871,7 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { public GlobValidator(final String glob) { this.glob = glob; } - private final Set matchingFieldsCache = new HashSet<>(); + private final Set matchingFieldsCache = new LinkedHashSet<>(); public String getFlParam() { return glob; } @@ -608,15 +887,15 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { final SolrInputDocument expected, final SolrDocument actual) { - final Set renamed = new HashSet<>(validators.size()); + final Set renamed = new LinkedHashSet<>(validators.size()); for (FlValidator v : validators) { - if (v instanceof RenameFieldValueValidator) { - renamed.add(((RenameFieldValueValidator)v).getRealFieldName()); + if (v instanceof SuppressRealFields) { + renamed.addAll(((SuppressRealFields)v).getSuppressedFields()); } } // every real field name matching the glob that is not renamed should be in the results - Set result = new HashSet<>(expected.getFieldNames().size()); + Set result = new LinkedHashSet<>(expected.getFieldNames().size()); for (String f : expected.getFieldNames()) { if ( matchesGlob(f) && (! renamed.contains(f) ) ) { result.add(f); @@ -650,6 +929,19 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { } } + /** explain should always be ignored when using RTG */ + private static class ExplainValidator extends NotIncludedValidator { + private final static String NAME = "explain"; + private final static String USAGE = "[" + NAME + "]"; + public ExplainValidator() { + super(USAGE); + } + public ExplainValidator(final String resultKey) { + super(USAGE, resultKey + ":" + USAGE); + } + public String getDefaultTransformerFactoryName() { return NAME; } + } + /** helper method for adding a random number (may be 0) of items from {@link #FL_VALIDATORS} */ private static void addRandomFlValidators(final Random r, final Set validators) { List copyToShuffle = new ArrayList<>(FL_VALIDATORS); @@ -672,4 +964,15 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase { } return result; } + + /** helper method for asserting an object is a non-null String can be parsed as an int */ + public static int assertParseInt(String msg, Object orig) { + assertNotNull(msg + ": is null", orig); + assertTrue(msg + ": is not a string: " + orig, orig instanceof String); + try { + return Integer.parseInt(orig.toString()); + } catch (NumberFormatException nfe) { + throw new AssertionError(msg + ": can't be parsed as a number: " + orig, nfe); + } + } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithDelegationTokens.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithDelegationTokens.java new file mode 100644 index 00000000000..a58ec8c45b2 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithDelegationTokens.java @@ -0,0 +1,402 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import junit.framework.Assert; +import org.apache.hadoop.util.Time; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.DelegationTokenRequest; +import org.apache.solr.client.solrj.response.DelegationTokenResponse; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import static org.apache.solr.security.HttpParamDelegationTokenPlugin.USER_PARAM; + +import org.apache.http.HttpStatus; +import org.apache.solr.security.HttpParamDelegationTokenPlugin; +import org.apache.solr.security.KerberosPlugin; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.lang.invoke.MethodHandles; +import java.util.HashSet; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test the delegation token support in the {@link org.apache.solr.security.KerberosPlugin}. + */ +@LuceneTestCase.Slow +public class TestSolrCloudWithDelegationTokens extends SolrTestCaseJ4 { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final int NUM_SERVERS = 2; + private static MiniSolrCloudCluster miniCluster; + private static HttpSolrClient solrClientPrimary; + private static HttpSolrClient solrClientSecondary; + + @BeforeClass + public static void startup() throws Exception { + System.setProperty("authenticationPlugin", HttpParamDelegationTokenPlugin.class.getName()); + System.setProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED, "true"); + System.setProperty("solr.kerberos.cookie.domain", "127.0.0.1"); + + miniCluster = new MiniSolrCloudCluster(NUM_SERVERS, createTempDir(), buildJettyConfig("/solr")); + JettySolrRunner runnerPrimary = miniCluster.getJettySolrRunners().get(0); + solrClientPrimary = + new HttpSolrClient.Builder(runnerPrimary.getBaseUrl().toString()) + .build(); + JettySolrRunner runnerSecondary = miniCluster.getJettySolrRunners().get(1); + solrClientSecondary = + new HttpSolrClient.Builder(runnerSecondary.getBaseUrl().toString()) + .build(); + } + + @AfterClass + public static void shutdown() throws Exception { + if (miniCluster != null) { + miniCluster.shutdown(); + } + miniCluster = null; + solrClientPrimary.close(); + solrClientPrimary = null; + solrClientSecondary.close(); + solrClientSecondary = null; + System.clearProperty("authenticationPlugin"); + System.clearProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED); + System.clearProperty("solr.kerberos.cookie.domain"); + } + + private String getDelegationToken(final String renewer, final String user, HttpSolrClient solrClient) throws Exception { + DelegationTokenRequest.Get get = new DelegationTokenRequest.Get(renewer) { + @Override + public SolrParams getParams() { + ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); + params.set(USER_PARAM, user); + return params; + } + }; + DelegationTokenResponse.Get getResponse = get.process(solrClient); + return getResponse.getDelegationToken(); + } + + private long renewDelegationToken(final String token, final int expectedStatusCode, + final String user, HttpSolrClient client) throws Exception { + DelegationTokenRequest.Renew renew = new DelegationTokenRequest.Renew(token) { + @Override + public SolrParams getParams() { + ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); + params.set(USER_PARAM, user); + return params; + } + + @Override + public Set getQueryParams() { + Set queryParams = super.getQueryParams(); + queryParams.add(USER_PARAM); + return queryParams; + } + }; + try { + DelegationTokenResponse.Renew renewResponse = renew.process(client); + assertEquals(HttpStatus.SC_OK, expectedStatusCode); + return renewResponse.getExpirationTime(); + } catch (HttpSolrClient.RemoteSolrException ex) { + assertEquals(expectedStatusCode, ex.code()); + return -1; + } + } + + private void cancelDelegationToken(String token, int expectedStatusCode, HttpSolrClient client) + throws Exception { + DelegationTokenRequest.Cancel cancel = new DelegationTokenRequest.Cancel(token); + try { + cancel.process(client); + assertEquals(HttpStatus.SC_OK, expectedStatusCode); + } catch (HttpSolrClient.RemoteSolrException ex) { + assertEquals(expectedStatusCode, ex.code()); + } + } + + private void doSolrRequest(String token, int expectedStatusCode, HttpSolrClient client) + throws Exception { + doSolrRequest(token, expectedStatusCode, client, 1); + } + + private void doSolrRequest(String token, int expectedStatusCode, HttpSolrClient client, int trials) + throws Exception { + int lastStatusCode = 0; + for (int i = 0; i < trials; ++i) { + lastStatusCode = getStatusCode(token, null, null, client); + if (lastStatusCode == expectedStatusCode) { + return; + } + Thread.sleep(1000); + } + assertEquals("Did not receieve excepted status code", expectedStatusCode, lastStatusCode); + } + + private SolrRequest getAdminRequest(final SolrParams params) { + return new CollectionAdminRequest.List() { + @Override + public SolrParams getParams() { + ModifiableSolrParams p = new ModifiableSolrParams(super.getParams()); + p.add(params); + return p; + } + }; + } + + private int getStatusCode(String token, final String user, final String op, HttpSolrClient client) + throws Exception { + HttpSolrClient delegationTokenServer = + new HttpSolrClient.Builder(client.getBaseURL().toString()) + .withDelegationToken(token) + .withResponseParser(client.getParser()) + .build(); + try { + ModifiableSolrParams p = new ModifiableSolrParams(); + if (user != null) p.set(USER_PARAM, user); + if (op != null) p.set("op", op); + SolrRequest req = getAdminRequest(p); + if (user != null || op != null) { + Set queryParams = new HashSet(); + if (user != null) queryParams.add(USER_PARAM); + if (op != null) queryParams.add("op"); + req.setQueryParams(queryParams); + } + try { + delegationTokenServer.request(req, null, null); + return HttpStatus.SC_OK; + } catch (HttpSolrClient.RemoteSolrException re) { + return re.code(); + } + } finally { + delegationTokenServer.close(); + } + } + + private void doSolrRequest(HttpSolrClient client, SolrRequest request, + int expectedStatusCode) throws Exception { + try { + client.request(request); + assertEquals(HttpStatus.SC_OK, expectedStatusCode); + } catch (HttpSolrClient.RemoteSolrException ex) { + assertEquals(expectedStatusCode, ex.code()); + } + } + + private void verifyTokenValid(String token) throws Exception { + // pass with token + doSolrRequest(token, HttpStatus.SC_OK, solrClientPrimary); + + // fail without token + doSolrRequest(null, ErrorCode.UNAUTHORIZED.code, solrClientPrimary); + + // pass with token on other server + doSolrRequest(token, HttpStatus.SC_OK, solrClientSecondary); + + // fail without token on other server + doSolrRequest(null, ErrorCode.UNAUTHORIZED.code, solrClientSecondary); + } + + /** + * Test basic Delegation Token get/verify + */ + @Test + public void testDelegationTokenVerify() throws Exception { + final String user = "bar"; + + // Get token + String token = getDelegationToken(null, user, solrClientPrimary); + assertNotNull(token); + verifyTokenValid(token); + } + + private void verifyTokenCancelled(String token, HttpSolrClient client) throws Exception { + // fail with token on both servers. If cancelToOtherURL is true, + // the request went to other url, so FORBIDDEN should be returned immediately. + // The cancelled token may take awhile to propogate to the standard url (via ZK). + // This is of course the opposite if cancelToOtherURL is false. + doSolrRequest(token, ErrorCode.FORBIDDEN.code, client, 10); + + // fail without token on both servers + doSolrRequest(null, ErrorCode.UNAUTHORIZED.code, solrClientPrimary); + doSolrRequest(null, ErrorCode.UNAUTHORIZED.code, solrClientSecondary); + } + + @Test + public void testDelegationTokenCancel() throws Exception { + { + // Get token + String token = getDelegationToken(null, "user", solrClientPrimary); + assertNotNull(token); + + // cancel token, note don't need to be authenticated to cancel (no user specified) + cancelDelegationToken(token, HttpStatus.SC_OK, solrClientPrimary); + verifyTokenCancelled(token, solrClientPrimary); + } + + { + // cancel token on different server from where we got it + String token = getDelegationToken(null, "user", solrClientPrimary); + assertNotNull(token); + + cancelDelegationToken(token, HttpStatus.SC_OK, solrClientSecondary); + verifyTokenCancelled(token, solrClientSecondary); + } + } + + @Test + public void testDelegationTokenCancelFail() throws Exception { + // cancel a bogus token + cancelDelegationToken("BOGUS", ErrorCode.NOT_FOUND.code, solrClientPrimary); + + { + // cancel twice, first on same server + String token = getDelegationToken(null, "bar", solrClientPrimary); + assertNotNull(token); + cancelDelegationToken(token, HttpStatus.SC_OK, solrClientPrimary); + cancelDelegationToken(token, ErrorCode.NOT_FOUND.code, solrClientSecondary); + cancelDelegationToken(token, ErrorCode.NOT_FOUND.code, solrClientPrimary); + } + + { + // cancel twice, first on other server + String token = getDelegationToken(null, "bar", solrClientPrimary); + assertNotNull(token); + cancelDelegationToken(token, HttpStatus.SC_OK, solrClientSecondary); + cancelDelegationToken(token, ErrorCode.NOT_FOUND.code, solrClientSecondary); + cancelDelegationToken(token, ErrorCode.NOT_FOUND.code, solrClientPrimary); + } + } + + private void verifyDelegationTokenRenew(String renewer, String user) + throws Exception { + { + // renew on same server + String token = getDelegationToken(renewer, user, solrClientPrimary); + assertNotNull(token); + long now = Time.now(); + assertTrue(renewDelegationToken(token, HttpStatus.SC_OK, user, solrClientPrimary) > now); + verifyTokenValid(token); + } + + { + // renew on different server + String token = getDelegationToken(renewer, user, solrClientPrimary); + assertNotNull(token); + long now = Time.now(); + assertTrue(renewDelegationToken(token, HttpStatus.SC_OK, user, solrClientSecondary) > now); + verifyTokenValid(token); + } + } + + @Test + public void testDelegationTokenRenew() throws Exception { + // test with specifying renewer + verifyDelegationTokenRenew("bar", "bar"); + + // test without specifying renewer + verifyDelegationTokenRenew(null, "bar"); + } + + @Test + public void testDelegationTokenRenewFail() throws Exception { + // don't set renewer and try to renew as an a different user + String token = getDelegationToken(null, "bar", solrClientPrimary); + assertNotNull(token); + renewDelegationToken(token, ErrorCode.FORBIDDEN.code, "foo", solrClientPrimary); + renewDelegationToken(token, ErrorCode.FORBIDDEN.code, "foo", solrClientSecondary); + + // set renewer and try to renew as different user + token = getDelegationToken("renewUser", "bar", solrClientPrimary); + assertNotNull(token); + renewDelegationToken(token, ErrorCode.FORBIDDEN.code, "notRenewUser", solrClientPrimary); + renewDelegationToken(token, ErrorCode.FORBIDDEN.code, "notRenewUser", solrClientSecondary); + } + + /** + * Test that a non-delegation-token "op" http param is handled correctly + */ + @Test + public void testDelegationOtherOp() throws Exception { + assertEquals(HttpStatus.SC_OK, getStatusCode(null, "bar", "someSolrOperation", solrClientPrimary)); + } + + @Test + public void testZNodePaths() throws Exception { + getDelegationToken(null, "bar", solrClientPrimary); + SolrZkClient zkClient = new SolrZkClient(miniCluster.getZkServer().getZkAddress(), 1000); + try { + assertTrue(zkClient.exists("/security/zkdtsm", true)); + assertTrue(zkClient.exists("/security/token", true)); + } finally { + zkClient.close(); + } + } + + /** + * Test HttpSolrServer's delegation token support + */ + @Test + public void testDelegationTokenSolrClient() throws Exception { + // Get token + String token = getDelegationToken(null, "bar", solrClientPrimary); + assertNotNull(token); + + SolrRequest request = getAdminRequest(new ModifiableSolrParams()); + + // test without token + HttpSolrClient ss = + new HttpSolrClient.Builder(solrClientPrimary.getBaseURL().toString()) + .withResponseParser(solrClientPrimary.getParser()) + .build(); + try { + doSolrRequest(ss, request, ErrorCode.UNAUTHORIZED.code); + } finally { + ss.close(); + } + + ss = new HttpSolrClient.Builder(solrClientPrimary.getBaseURL().toString()) + .withDelegationToken(token) + .withResponseParser(solrClientPrimary.getParser()) + .build(); + try { + // test with token via property + doSolrRequest(ss, request, HttpStatus.SC_OK); + + // test with param -- should throw an exception + ModifiableSolrParams tokenParam = new ModifiableSolrParams(); + tokenParam.set("delegation", "invalidToken"); + try { + doSolrRequest(ss, getAdminRequest(tokenParam), ErrorCode.FORBIDDEN.code); + Assert.fail("Expected exception"); + } catch (IllegalArgumentException ex) {} + } finally { + ss.close(); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java index 6ac2254731b..c505b519c88 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java @@ -17,15 +17,12 @@ package org.apache.solr.cloud; import java.io.File; +import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Locale; import java.util.Properties; -import javax.security.auth.login.Configuration; - import org.apache.commons.io.FileUtils; -import org.apache.hadoop.minikdc.MiniKdc; import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.util.Constants; import org.apache.lucene.util.LuceneTestCase; @@ -49,6 +46,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; @@ -67,7 +66,7 @@ import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; @LuceneTestCase.SuppressSysoutChecks(bugUrl = "Solr logs to JUL") public class TestSolrCloudWithKerberosAlt extends LuceneTestCase { - private final Configuration originalConfig = Configuration.getConfiguration(); + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); protected final int NUM_SERVERS; protected final int NUM_SHARDS; protected final int REPLICATION_FACTOR; @@ -78,10 +77,8 @@ public class TestSolrCloudWithKerberosAlt extends LuceneTestCase { REPLICATION_FACTOR = 1; } - private MiniKdc kdc; + private KerberosTestServices kerberosTestServices; - private Locale savedLocale; // in case locale is broken and we need to fill in a working locale - @Rule public TestRule solrTestRules = RuleChain .outerRule(new SystemPropertiesRestoreRule()); @@ -98,7 +95,6 @@ public class TestSolrCloudWithKerberosAlt extends LuceneTestCase { @Override public void setUp() throws Exception { - savedLocale = KerberosTestUtil.overrideLocaleIfNotSpportedByMiniKdc(); super.setUp(); setupMiniKdc(); } @@ -106,12 +102,15 @@ public class TestSolrCloudWithKerberosAlt extends LuceneTestCase { private void setupMiniKdc() throws Exception { System.setProperty("solr.jaas.debug", "true"); String kdcDir = createTempDir()+File.separator+"minikdc"; - kdc = KerberosTestUtil.getKdc(new File(kdcDir)); - File keytabFile = new File(kdcDir, "keytabs"); - String solrServerPrincipal = "HTTP/127.0.0.1"; String solrClientPrincipal = "solr"; - kdc.start(); - kdc.createPrincipal(keytabFile, solrServerPrincipal, solrClientPrincipal); + File keytabFile = new File(kdcDir, "keytabs"); + kerberosTestServices = KerberosTestServices.builder() + .withKdc(new File(kdcDir)) + .withJaasConfiguration(solrClientPrincipal, keytabFile, "SolrClient") + .build(); + String solrServerPrincipal = "HTTP/127.0.0.1"; + kerberosTestServices.start(); + kerberosTestServices.getKdc().createPrincipal(keytabFile, solrServerPrincipal, solrClientPrincipal); String jaas = "SolrClient {\n" + " com.sun.security.auth.module.Krb5LoginModule required\n" @@ -124,9 +123,6 @@ public class TestSolrCloudWithKerberosAlt extends LuceneTestCase { + " principal=\"" + solrClientPrincipal + "\";\n" + "};"; - Configuration conf = new KerberosTestUtil.JaasConfiguration(solrClientPrincipal, keytabFile, "SolrClient"); - Configuration.setConfiguration(conf); - String jaasFilePath = kdcDir+File.separator+"jaas-client.conf"; FileUtils.write(new File(jaasFilePath), jaas, StandardCharsets.UTF_8); System.setProperty("java.security.auth.login.config", jaasFilePath); @@ -135,6 +131,9 @@ public class TestSolrCloudWithKerberosAlt extends LuceneTestCase { System.setProperty("solr.kerberos.principal", solrServerPrincipal); System.setProperty("solr.kerberos.keytab", keytabFile.getAbsolutePath()); System.setProperty("authenticationPlugin", "org.apache.solr.security.KerberosPlugin"); + boolean enableDt = random().nextBoolean(); + log.info("Enable delegation token: " + enableDt); + System.setProperty("solr.kerberos.delegation.token.enabled", new Boolean(enableDt).toString()); // Extracts 127.0.0.1 from HTTP/127.0.0.1@EXAMPLE.COM System.setProperty("solr.kerberos.name.rules", "RULE:[1:$1@$0](.*EXAMPLE.COM)s/@.*//" + "\nRULE:[2:$2@$0](.*EXAMPLE.COM)s/@.*//" @@ -240,11 +239,7 @@ public class TestSolrCloudWithKerberosAlt extends LuceneTestCase { System.clearProperty("authenticationPlugin"); System.clearProperty("solr.kerberos.name.rules"); System.clearProperty("solr.jaas.debug"); - Configuration.setConfiguration(this.originalConfig); - if (kdc != null) { - kdc.stop(); - } - Locale.setDefault(savedLocale); + kerberosTestServices.stop(); super.tearDown(); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java new file mode 100644 index 00000000000..71107eed5e3 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import javax.servlet.http.HttpServletRequest; +import java.io.File; +import java.net.InetAddress; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.conf.Configuration; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.admin.CollectionsHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.security.HttpParamDelegationTokenPlugin; +import org.apache.solr.security.KerberosPlugin; +import org.apache.solr.servlet.SolrRequestParsers; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.apache.solr.security.HttpParamDelegationTokenPlugin.USER_PARAM; +import static org.apache.solr.security.HttpParamDelegationTokenPlugin.REMOTE_HOST_PARAM; +import static org.apache.solr.security.HttpParamDelegationTokenPlugin.REMOTE_ADDRESS_PARAM; + +public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 { + private static final int NUM_SERVERS = 2; + private static MiniSolrCloudCluster miniCluster; + private static SolrClient solrClient; + + private static String getUsersFirstGroup() throws Exception { + org.apache.hadoop.security.Groups hGroups = + new org.apache.hadoop.security.Groups(new Configuration()); + String group = "*"; // accept any group if a group can't be found + try { + List g = hGroups.getGroups(System.getProperty("user.name")); + if (g != null && g.size() > 0) { + group = g.get(0); + } + } catch (NullPointerException npe) { + // if user/group doesn't exist on test box + } + return group; + } + + private static Map getImpersonatorSettings() throws Exception { + Map filterProps = new TreeMap(); + filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "noGroups.hosts", "*"); + filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostAnyUser.groups", "*"); + filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostAnyUser.hosts", "*"); + filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "wrongHost.hosts", "1.1.1.1.1.1"); + filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "wrongHost.groups", "*"); + filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "noHosts.groups", "*"); + filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "localHostAnyGroup.groups", "*"); + InetAddress loopback = InetAddress.getLoopbackAddress(); + filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "localHostAnyGroup.hosts", + loopback.getCanonicalHostName() + "," + loopback.getHostName() + "," + loopback.getHostAddress()); + filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostUsersGroup.groups", getUsersFirstGroup()); + filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostUsersGroup.hosts", "*"); + filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "bogusGroup.groups", "__some_bogus_group"); + filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "bogusGroup.hosts", "*"); + return filterProps; + } + + @BeforeClass + public static void startup() throws Exception { + System.setProperty("authenticationPlugin", HttpParamDelegationTokenPlugin.class.getName()); + System.setProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED, "true"); + + System.setProperty("solr.kerberos.cookie.domain", "127.0.0.1"); + Map impSettings = getImpersonatorSettings(); + for (Map.Entry entry : impSettings.entrySet()) { + System.setProperty(entry.getKey(), entry.getValue()); + } + System.setProperty("solr.test.sys.prop1", "propone"); + System.setProperty("solr.test.sys.prop2", "proptwo"); + + SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(true); + String solrXml = MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML.replace("", + " " + ImpersonatorCollectionsHandler.class.getName() + "\n" + + ""); + + miniCluster = new MiniSolrCloudCluster(NUM_SERVERS, createTempDir(), solrXml, buildJettyConfig("/solr")); + JettySolrRunner runner = miniCluster.getJettySolrRunners().get(0); + solrClient = new HttpSolrClient.Builder(runner.getBaseUrl().toString()).build(); + } + + /** + * Verify that impersonator info is preserved in the request + */ + public static class ImpersonatorCollectionsHandler extends CollectionsHandler { + public static AtomicBoolean called = new AtomicBoolean(false); + + public ImpersonatorCollectionsHandler() { super(); } + + public ImpersonatorCollectionsHandler(final CoreContainer coreContainer) { + super(coreContainer); + } + + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { + called.set(true); + super.handleRequestBody(req, rsp); + String doAs = req.getParams().get(KerberosPlugin.IMPERSONATOR_DO_AS_HTTP_PARAM); + if (doAs != null) { + HttpServletRequest httpRequest = (HttpServletRequest)req.getContext().get("httpRequest"); + assertNotNull(httpRequest); + String user = (String)httpRequest.getAttribute(USER_PARAM); + assertNotNull(user); + assertEquals(user, httpRequest.getAttribute(KerberosPlugin.IMPERSONATOR_USER_NAME)); + } + } + } + + @Before + public void clearCalledIndicator() throws Exception { + ImpersonatorCollectionsHandler.called.set(false); + } + + @AfterClass + public static void shutdown() throws Exception { + if (miniCluster != null) { + miniCluster.shutdown(); + } + miniCluster = null; + solrClient.close(); + solrClient = null; + System.clearProperty("authenticationPlugin"); + System.clearProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED); + System.clearProperty("solr.kerberos.cookie.domain"); + Map impSettings = getImpersonatorSettings(); + for (Map.Entry entry : impSettings.entrySet()) { + System.clearProperty(entry.getKey()); + } + System.clearProperty("solr.test.sys.prop1"); + System.clearProperty("solr.test.sys.prop2"); + SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(false); + } + + private void create1ShardCollection(String name, String config, MiniSolrCloudCluster solrCluster) throws Exception { + CollectionAdminResponse response; + CollectionAdminRequest.Create create = new CollectionAdminRequest.Create() { + @Override + public SolrParams getParams() { + ModifiableSolrParams msp = new ModifiableSolrParams(super.getParams()); + msp.set(USER_PARAM, "user"); + return msp; + } + }; + create.setConfigName(config); + create.setCollectionName(name); + create.setNumShards(1); + create.setReplicationFactor(1); + create.setMaxShardsPerNode(1); + response = create.process(solrCluster.getSolrClient()); + + if (response.getStatus() != 0 || response.getErrorMessages() != null) { + fail("Could not create collection. Response" + response.toString()); + } + ZkStateReader zkStateReader = solrCluster.getSolrClient().getZkStateReader(); + AbstractDistribZkTestBase.waitForRecoveriesToFinish(name, zkStateReader, false, true, 100); + } + + private SolrRequest getProxyRequest(String user, String doAs) { + return getProxyRequest(user, doAs, null); + } + + private SolrRequest getProxyRequest(String user, String doAs, String remoteHost) { + return getProxyRequest(user, doAs, remoteHost, null); + } + + private SolrRequest getProxyRequest(String user, String doAs, String remoteHost, String remoteAddress) { + return new CollectionAdminRequest.List() { + @Override + public SolrParams getParams() { + ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); + params.set(USER_PARAM, user); + params.set(KerberosPlugin.IMPERSONATOR_DO_AS_HTTP_PARAM, doAs); + if (remoteHost != null) params.set(REMOTE_HOST_PARAM, remoteHost); + if (remoteAddress != null) params.set(REMOTE_ADDRESS_PARAM, remoteAddress); + return params; + } + }; + } + + private String getExpectedGroupExMsg(String user, String doAs) { + return "User: " + user + " is not allowed to impersonate " + doAs; + } + + private String getExpectedHostExMsg(String user) { + return "Unauthorized connection for super-user: " + user; + } + + @Test + public void testProxyNoConfigGroups() throws Exception { + try { + solrClient.request(getProxyRequest("noGroups","bar")); + fail("Expected RemoteSolrException"); + } + catch (HttpSolrClient.RemoteSolrException ex) { + assertTrue(ex.getMessage().contains(getExpectedGroupExMsg("noGroups", "bar"))); + } + } + + @Test + public void testProxyWrongHost() throws Exception { + try { + solrClient.request(getProxyRequest("wrongHost","bar")); + fail("Expected RemoteSolrException"); + } + catch (HttpSolrClient.RemoteSolrException ex) { + assertTrue(ex.getMessage().contains(getExpectedHostExMsg("wrongHost"))); + } + } + + @Test + public void testProxyNoConfigHosts() throws Exception { + try { + solrClient.request(getProxyRequest("noHosts","bar")); + fail("Expected RemoteSolrException"); + } + catch (HttpSolrClient.RemoteSolrException ex) { + // FixMe: this should return an exception about the host being invalid, + // but a bug (HADOOP-11077) causes an NPE instead. + //assertTrue(ex.getMessage().contains(getExpectedHostExMsg("noHosts"))); + } + } + + @Test + public void testProxyValidateAnyHostAnyUser() throws Exception { + solrClient.request(getProxyRequest("anyHostAnyUser", "bar", null)); + assertTrue(ImpersonatorCollectionsHandler.called.get()); + } + + @Test + public void testProxyInvalidProxyUser() throws Exception { + try { + // wrong direction, should fail + solrClient.request(getProxyRequest("bar","anyHostAnyUser")); + fail("Expected RemoteSolrException"); + } + catch (HttpSolrClient.RemoteSolrException ex) { + assertTrue(ex.getMessage().contains(getExpectedGroupExMsg("bar", "anyHostAnyUser"))); + } + } + + @Test + public void testProxyValidateHost() throws Exception { + solrClient.request(getProxyRequest("localHostAnyGroup", "bar")); + assertTrue(ImpersonatorCollectionsHandler.called.get()); + } + + + + @Test + public void testProxyValidateGroup() throws Exception { + solrClient.request(getProxyRequest("anyHostUsersGroup", System.getProperty("user.name"), null)); + assertTrue(ImpersonatorCollectionsHandler.called.get()); + } + + @Test + public void testProxyUnknownRemote() throws Exception { + try { + // Use a reserved ip address + String nonProxyUserConfiguredIpAddress = "255.255.255.255"; + solrClient.request(getProxyRequest("localHostAnyGroup", "bar", "unknownhost.bar.foo", nonProxyUserConfiguredIpAddress)); + fail("Expected RemoteSolrException"); + } + catch (HttpSolrClient.RemoteSolrException ex) { + assertTrue(ex.getMessage().contains(getExpectedHostExMsg("localHostAnyGroup"))); + } + } + + @Test + public void testProxyInvalidRemote() throws Exception { + try { + String invalidIpAddress = "-127.-128"; + solrClient.request(getProxyRequest("localHostAnyGroup","bar", "[ff01::114]", invalidIpAddress)); + fail("Expected RemoteSolrException"); + } + catch (HttpSolrClient.RemoteSolrException ex) { + assertTrue(ex.getMessage().contains(getExpectedHostExMsg("localHostAnyGroup"))); + } + } + + @Test + public void testProxyInvalidGroup() throws Exception { + try { + solrClient.request(getProxyRequest("bogusGroup","bar", null)); + fail("Expected RemoteSolrException"); + } + catch (HttpSolrClient.RemoteSolrException ex) { + assertTrue(ex.getMessage().contains(getExpectedGroupExMsg("bogusGroup", "bar"))); + } + } + + @Test + public void testProxyNullProxyUser() throws Exception { + try { + solrClient.request(getProxyRequest("","bar")); + fail("Expected RemoteSolrException"); + } + catch (HttpSolrClient.RemoteSolrException ex) { + // this exception is specific to our implementation, don't check a specific message. + } + } + + @Test + public void testForwarding() throws Exception { + String collectionName = "forwardingCollection"; + File configDir = getFile("solr").toPath().resolve("collection1/conf").toFile(); + miniCluster.uploadConfigDir(configDir, "conf1"); + create1ShardCollection(collectionName, "conf1", miniCluster); + + // try a command to each node, one of them must be forwarded + for (JettySolrRunner jetty : miniCluster.getJettySolrRunners()) { + HttpSolrClient client = + new HttpSolrClient.Builder(jetty.getBaseUrl().toString() + "/" + collectionName).build(); + try { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("q", "*:*"); + params.set(USER_PARAM, "user"); + client.query(params); + } finally { + client.close(); + } + } + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java b/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java index 80bb98aef6d..fc6d18c0605 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java @@ -97,6 +97,14 @@ public class TestStressCloudBlindAtomicUpdates extends SolrCloudTestCase { * larger index is used (so tested docs are more likeely to be spread out in multiple segments) */ private static int DOC_ID_INCR; + + /** + * The TestInjection configuration to be used for the current test method. + * + * Value is set by {@link #clearCloudCollection}, and used by {@link #startTestInjection} -- but only once + * initial index seeding has finished (we're focusing on testing atomic updates, not basic indexing). + */ + private String testInjection = null; @BeforeClass private static void createMiniSolrCloudCluster() throws Exception { @@ -161,9 +169,17 @@ public class TestStressCloudBlindAtomicUpdates extends SolrCloudTestCase { assertEquals(0, CLOUD_CLIENT.optimize().getStatus()); TestInjection.reset(); - + final int injectionPercentage = (int)Math.ceil(atLeast(1) / 2); - final String testInjection = usually() ? "false:0" : ("true:" + injectionPercentage); + testInjection = usually() ? "false:0" : ("true:" + injectionPercentage); + } + + /** + * Assigns {@link #testInjection} to various TestInjection variables. Calling this + * method multiple times in the same method should always result in the same setting being applied + * (even if {@link TestInjection#reset} was called in between. + */ + private void startTestInjection() { log.info("TestInjection: fail replica, update pause, tlog pauses: " + testInjection); TestInjection.failReplicaRequests = testInjection; TestInjection.updateLogReplayRandomPause = testInjection; @@ -249,10 +265,13 @@ public class TestStressCloudBlindAtomicUpdates extends SolrCloudTestCase { // sanity check index contents + waitForRecoveriesToFinish(CLOUD_CLIENT); assertEquals(0, CLOUD_CLIENT.commit().getStatus()); assertEquals(numDocsInIndex, CLOUD_CLIENT.query(params("q", "*:*")).getResults().getNumFound()); + startTestInjection(); + // spin up parallel workers to hammer updates List> results = new ArrayList>(NUM_THREADS); for (int workerId = 0; workerId < NUM_THREADS; workerId++) { @@ -301,7 +320,7 @@ public class TestStressCloudBlindAtomicUpdates extends SolrCloudTestCase { // sometimes include an fq on the expected value to ensure the updated values // are "visible" for searching final SolrParams p = (0 != TestUtil.nextInt(random(), 0,15)) - ? params() : params("fq",numericFieldName + ":" + expect); + ? params() : params("fq",numericFieldName + ":\"" + expect + "\""); SolrDocument doc = getRandClient(random()).getById(docId, p); final boolean foundWithFilter = (null != doc); diff --git a/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java index 31919a81af8..95422fa04b2 100644 --- a/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java @@ -21,6 +21,7 @@ import java.lang.invoke.MethodHandles; import java.nio.charset.Charset; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider; import org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider; @@ -76,6 +77,8 @@ public class VMParamsZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); zkClient.create("/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); zkClient.makePath("/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + + zkClient.create(SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); zkClient.close(); clearSecuritySystemProperties(); @@ -106,7 +109,9 @@ public class VMParamsZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - doTest(zkClient, false, false, false, false, false); + doTest(zkClient, + false, false, false, false, false, + false, false, false, false, false); } finally { zkClient.close(); } @@ -118,7 +123,9 @@ public class VMParamsZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - doTest(zkClient, false, false, false, false, false); + doTest(zkClient, + false, false, false, false, false, + false, false, false, false, false); } finally { zkClient.close(); } @@ -130,7 +137,9 @@ public class VMParamsZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - doTest(zkClient, true, true, true, true, true); + doTest(zkClient, + true, true, true, true, true, + true, true, true, true, true); } finally { zkClient.close(); } @@ -142,17 +151,23 @@ public class VMParamsZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); try { - doTest(zkClient, true, true, false, false, false); + doTest(zkClient, + true, true, false, false, false, + false, false, false, false, false); } finally { zkClient.close(); } } - protected static void doTest(SolrZkClient zkClient, boolean getData, boolean list, boolean create, boolean setData, boolean delete) throws Exception { + protected static void doTest( + SolrZkClient zkClient, + boolean getData, boolean list, boolean create, boolean setData, boolean delete, + boolean secureGet, boolean secureList, boolean secureCreate, boolean secureSet, boolean secureDelete) throws Exception { doTest(zkClient, "/protectedCreateNode", getData, list, create, setData, delete); doTest(zkClient, "/protectedMakePathNode", getData, list, create, setData, delete); doTest(zkClient, "/unprotectedCreateNode", true, true, true, true, delete); doTest(zkClient, "/unprotectedMakePathNode", true, true, true, true, delete); + doTest(zkClient, SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, secureGet, secureList, secureCreate, secureSet, secureDelete); } protected static void doTest(SolrZkClient zkClient, String path, boolean getData, boolean list, boolean create, boolean setData, boolean delete) throws Exception { diff --git a/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java b/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java index 86aac4d1224..85dbf4aba72 100644 --- a/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateWriterTest.java @@ -57,68 +57,69 @@ public class ZkStateWriterTest extends SolrTestCaseJ4 { zkClient = new SolrZkClient(server.getZkAddress(), OverseerTest.DEFAULT_CONNECTION_TIMEOUT); ZkController.createClusterZkNodes(zkClient); - ZkStateReader reader = new ZkStateReader(zkClient); - reader.createClusterStateWatchersAndUpdate(); + try (ZkStateReader reader = new ZkStateReader(zkClient)) { + reader.createClusterStateWatchersAndUpdate(); - ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats()); + ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats()); - assertFalse("Deletes can always be batched", writer.maybeFlushBefore(new ZkWriteCommand("xyz", null))); - assertFalse("Deletes can always be batched", writer.maybeFlushAfter(new ZkWriteCommand("xyz", null))); + assertFalse("Deletes can always be batched", writer.maybeFlushBefore(new ZkWriteCommand("xyz", null))); + assertFalse("Deletes can always be batched", writer.maybeFlushAfter(new ZkWriteCommand("xyz", null))); - zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true); - zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c2", true); + zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true); + zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c2", true); - // create new collection with stateFormat = 2 - ZkWriteCommand c1 = new ZkWriteCommand("c1", - new DocCollection("c1", new HashMap<>(), new HashMap<>(), DocRouter.DEFAULT, 0, ZkStateReader.COLLECTIONS_ZKNODE + "/c1")); - assertFalse("First requests can always be batched", writer.maybeFlushBefore(c1)); + // create new collection with stateFormat = 2 + ZkWriteCommand c1 = new ZkWriteCommand("c1", + new DocCollection("c1", new HashMap<>(), new HashMap<>(), DocRouter.DEFAULT, 0, ZkStateReader.COLLECTIONS_ZKNODE + "/c1")); + assertFalse("First requests can always be batched", writer.maybeFlushBefore(c1)); - ClusterState clusterState = writer.enqueueUpdate(reader.getClusterState(), c1, null); + ClusterState clusterState = writer.enqueueUpdate(reader.getClusterState(), c1, null); - ZkWriteCommand c2 = new ZkWriteCommand("c2", - new DocCollection("c2", new HashMap<>(), new HashMap<>(), DocRouter.DEFAULT, 0, ZkStateReader.COLLECTIONS_ZKNODE + "/c2")); - assertTrue("Different (new) collection create cannot be batched together with another create", writer.maybeFlushBefore(c2)); + ZkWriteCommand c2 = new ZkWriteCommand("c2", + new DocCollection("c2", new HashMap<>(), new HashMap<>(), DocRouter.DEFAULT, 0, ZkStateReader.COLLECTIONS_ZKNODE + "/c2")); + assertTrue("Different (new) collection create cannot be batched together with another create", writer.maybeFlushBefore(c2)); - // simulate three state changes on same collection, all should be batched together before - assertFalse(writer.maybeFlushBefore(c1)); - assertFalse(writer.maybeFlushBefore(c1)); - assertFalse(writer.maybeFlushBefore(c1)); - // and after too - assertFalse(writer.maybeFlushAfter(c1)); - assertFalse(writer.maybeFlushAfter(c1)); - assertFalse(writer.maybeFlushAfter(c1)); + // simulate three state changes on same collection, all should be batched together before + assertFalse(writer.maybeFlushBefore(c1)); + assertFalse(writer.maybeFlushBefore(c1)); + assertFalse(writer.maybeFlushBefore(c1)); + // and after too + assertFalse(writer.maybeFlushAfter(c1)); + assertFalse(writer.maybeFlushAfter(c1)); + assertFalse(writer.maybeFlushAfter(c1)); - // simulate three state changes on two different collections with stateFormat=2, none should be batched - assertFalse(writer.maybeFlushBefore(c1)); - // flushAfter has to be called as it updates the internal batching related info - assertFalse(writer.maybeFlushAfter(c1)); - assertTrue(writer.maybeFlushBefore(c2)); - assertFalse(writer.maybeFlushAfter(c2)); - assertTrue(writer.maybeFlushBefore(c1)); - assertFalse(writer.maybeFlushAfter(c1)); + // simulate three state changes on two different collections with stateFormat=2, none should be batched + assertFalse(writer.maybeFlushBefore(c1)); + // flushAfter has to be called as it updates the internal batching related info + assertFalse(writer.maybeFlushAfter(c1)); + assertTrue(writer.maybeFlushBefore(c2)); + assertFalse(writer.maybeFlushAfter(c2)); + assertTrue(writer.maybeFlushBefore(c1)); + assertFalse(writer.maybeFlushAfter(c1)); - // create a collection in stateFormat = 1 i.e. inside the main cluster state - ZkWriteCommand c3 = new ZkWriteCommand("c3", - new DocCollection("c3", new HashMap<>(), new HashMap<>(), DocRouter.DEFAULT, 0, ZkStateReader.CLUSTER_STATE)); - clusterState = writer.enqueueUpdate(clusterState, c3, null); + // create a collection in stateFormat = 1 i.e. inside the main cluster state + ZkWriteCommand c3 = new ZkWriteCommand("c3", + new DocCollection("c3", new HashMap<>(), new HashMap<>(), DocRouter.DEFAULT, 0, ZkStateReader.CLUSTER_STATE)); + clusterState = writer.enqueueUpdate(clusterState, c3, null); - // simulate three state changes in c3, all should be batched - for (int i=0; i<3; i++) { + // simulate three state changes in c3, all should be batched + for (int i = 0; i < 3; i++) { + assertFalse(writer.maybeFlushBefore(c3)); + assertFalse(writer.maybeFlushAfter(c3)); + } + + // simulate state change in c3 (stateFormat=1) interleaved with state changes from c1,c2 (stateFormat=2) + // none should be batched together assertFalse(writer.maybeFlushBefore(c3)); assertFalse(writer.maybeFlushAfter(c3)); + assertTrue("different stateFormat, should be flushed", writer.maybeFlushBefore(c1)); + assertFalse(writer.maybeFlushAfter(c1)); + assertTrue("different stateFormat, should be flushed", writer.maybeFlushBefore(c3)); + assertFalse(writer.maybeFlushAfter(c3)); + assertTrue("different stateFormat, should be flushed", writer.maybeFlushBefore(c2)); + assertFalse(writer.maybeFlushAfter(c2)); } - // simulate state change in c3 (stateFormat=1) interleaved with state changes from c1,c2 (stateFormat=2) - // none should be batched together - assertFalse(writer.maybeFlushBefore(c3)); - assertFalse(writer.maybeFlushAfter(c3)); - assertTrue("different stateFormat, should be flushed", writer.maybeFlushBefore(c1)); - assertFalse(writer.maybeFlushAfter(c1)); - assertTrue("different stateFormat, should be flushed", writer.maybeFlushBefore(c3)); - assertFalse(writer.maybeFlushAfter(c3)); - assertTrue("different stateFormat, should be flushed", writer.maybeFlushBefore(c2)); - assertFalse(writer.maybeFlushAfter(c2)); - } finally { IOUtils.close(zkClient); server.shutdown(); @@ -140,24 +141,25 @@ public class ZkStateWriterTest extends SolrTestCaseJ4 { zkClient = new SolrZkClient(server.getZkAddress(), OverseerTest.DEFAULT_CONNECTION_TIMEOUT); ZkController.createClusterZkNodes(zkClient); - ZkStateReader reader = new ZkStateReader(zkClient); - reader.createClusterStateWatchersAndUpdate(); + try (ZkStateReader reader = new ZkStateReader(zkClient)) { + reader.createClusterStateWatchersAndUpdate(); - ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats()); + ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats()); - zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true); + zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true); - // create new collection with stateFormat = 1 - ZkWriteCommand c1 = new ZkWriteCommand("c1", - new DocCollection("c1", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.CLUSTER_STATE)); + // create new collection with stateFormat = 1 + ZkWriteCommand c1 = new ZkWriteCommand("c1", + new DocCollection("c1", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.CLUSTER_STATE)); - ClusterState clusterState = writer.enqueueUpdate(reader.getClusterState(), c1, null); - writer.writePendingUpdates(); + ClusterState clusterState = writer.enqueueUpdate(reader.getClusterState(), c1, null); + writer.writePendingUpdates(); - Map map = (Map) Utils.fromJSON(zkClient.getData("/clusterstate.json", null, null, true)); - assertNotNull(map.get("c1")); - boolean exists = zkClient.exists(ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json", true); - assertFalse(exists); + Map map = (Map) Utils.fromJSON(zkClient.getData("/clusterstate.json", null, null, true)); + assertNotNull(map.get("c1")); + boolean exists = zkClient.exists(ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json", true); + assertFalse(exists); + } } finally { IOUtils.close(zkClient); @@ -181,24 +183,25 @@ public class ZkStateWriterTest extends SolrTestCaseJ4 { zkClient = new SolrZkClient(server.getZkAddress(), OverseerTest.DEFAULT_CONNECTION_TIMEOUT); ZkController.createClusterZkNodes(zkClient); - ZkStateReader reader = new ZkStateReader(zkClient); - reader.createClusterStateWatchersAndUpdate(); + try (ZkStateReader reader = new ZkStateReader(zkClient)) { + reader.createClusterStateWatchersAndUpdate(); - ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats()); + ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats()); - zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true); + zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true); - // create new collection with stateFormat = 2 - ZkWriteCommand c1 = new ZkWriteCommand("c1", - new DocCollection("c1", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json")); + // create new collection with stateFormat = 2 + ZkWriteCommand c1 = new ZkWriteCommand("c1", + new DocCollection("c1", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json")); - ClusterState clusterState = writer.enqueueUpdate(reader.getClusterState(), c1, null); - writer.writePendingUpdates(); + ClusterState clusterState = writer.enqueueUpdate(reader.getClusterState(), c1, null); + writer.writePendingUpdates(); - Map map = (Map) Utils.fromJSON(zkClient.getData("/clusterstate.json", null, null, true)); - assertNull(map.get("c1")); - map = (Map) Utils.fromJSON(zkClient.getData(ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json", null, null, true)); - assertNotNull(map.get("c1")); + Map map = (Map) Utils.fromJSON(zkClient.getData("/clusterstate.json", null, null, true)); + assertNull(map.get("c1")); + map = (Map) Utils.fromJSON(zkClient.getData(ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json", null, null, true)); + assertNotNull(map.get("c1")); + } } finally { IOUtils.close(zkClient); @@ -224,63 +227,64 @@ public class ZkStateWriterTest extends SolrTestCaseJ4 { zkClient = new SolrZkClient(server.getZkAddress(), OverseerTest.DEFAULT_CONNECTION_TIMEOUT); ZkController.createClusterZkNodes(zkClient); - ZkStateReader reader = new ZkStateReader(zkClient); - reader.createClusterStateWatchersAndUpdate(); + try (ZkStateReader reader = new ZkStateReader(zkClient)) { + reader.createClusterStateWatchersAndUpdate(); - ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats()); + ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats()); - zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true); - zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c2", true); + zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true); + zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c2", true); - // create collection 1 with stateFormat = 1 - ZkWriteCommand c1 = new ZkWriteCommand("c1", - new DocCollection("c1", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.CLUSTER_STATE)); - writer.enqueueUpdate(reader.getClusterState(), c1, null); - writer.writePendingUpdates(); - - reader.forceUpdateCollection("c1"); - reader.forceUpdateCollection("c2"); - ClusterState clusterState = reader.getClusterState(); // keep a reference to the current cluster state object - assertTrue(clusterState.hasCollection("c1")); - assertFalse(clusterState.hasCollection("c2")); - - // Simulate an external modification to /clusterstate.json - byte[] data = zkClient.getData("/clusterstate.json", null, null, true); - zkClient.setData("/clusterstate.json", data, true); - - // enqueue another c1 so that ZkStateWriter has pending updates - writer.enqueueUpdate(clusterState, c1, null); - assertTrue(writer.hasPendingUpdates()); - - // create collection 2 with stateFormat = 1 - ZkWriteCommand c2 = new ZkWriteCommand("c2", - new DocCollection("c2", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.getCollectionPath("c2"))); - - try { - writer.enqueueUpdate(clusterState, c2, null); // we are sending in the old cluster state object - fail("Enqueue should not have succeeded"); - } catch (KeeperException.BadVersionException bve) { - // expected - } - - try { - writer.enqueueUpdate(reader.getClusterState(), c2, null); - fail("enqueueUpdate after BadVersionException should not have succeeded"); - } catch (IllegalStateException e) { - // expected - } - - try { + // create collection 1 with stateFormat = 1 + ZkWriteCommand c1 = new ZkWriteCommand("c1", + new DocCollection("c1", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.CLUSTER_STATE)); + writer.enqueueUpdate(reader.getClusterState(), c1, null); writer.writePendingUpdates(); - fail("writePendingUpdates after BadVersionException should not have succeeded"); - } catch (IllegalStateException e) { - // expected - } + reader.forceUpdateCollection("c1"); + reader.forceUpdateCollection("c2"); + ClusterState clusterState = reader.getClusterState(); // keep a reference to the current cluster state object + assertTrue(clusterState.hasCollection("c1")); + assertFalse(clusterState.hasCollection("c2")); + + // Simulate an external modification to /clusterstate.json + byte[] data = zkClient.getData("/clusterstate.json", null, null, true); + zkClient.setData("/clusterstate.json", data, true); + + // enqueue another c1 so that ZkStateWriter has pending updates + writer.enqueueUpdate(clusterState, c1, null); + assertTrue(writer.hasPendingUpdates()); + + // create collection 2 with stateFormat = 1 + ZkWriteCommand c2 = new ZkWriteCommand("c2", + new DocCollection("c2", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.getCollectionPath("c2"))); + + try { + writer.enqueueUpdate(clusterState, c2, null); // we are sending in the old cluster state object + fail("Enqueue should not have succeeded"); + } catch (KeeperException.BadVersionException bve) { + // expected + } + + try { + writer.enqueueUpdate(reader.getClusterState(), c2, null); + fail("enqueueUpdate after BadVersionException should not have succeeded"); + } catch (IllegalStateException e) { + // expected + } + + try { + writer.writePendingUpdates(); + fail("writePendingUpdates after BadVersionException should not have succeeded"); + } catch (IllegalStateException e) { + // expected + } + } } finally { IOUtils.close(zkClient); server.shutdown(); } + } public void testExternalModificationToStateFormat2() throws Exception { @@ -298,68 +302,69 @@ public class ZkStateWriterTest extends SolrTestCaseJ4 { zkClient = new SolrZkClient(server.getZkAddress(), OverseerTest.DEFAULT_CONNECTION_TIMEOUT); ZkController.createClusterZkNodes(zkClient); - ZkStateReader reader = new ZkStateReader(zkClient); - reader.createClusterStateWatchersAndUpdate(); + try (ZkStateReader reader = new ZkStateReader(zkClient)) { + reader.createClusterStateWatchersAndUpdate(); - ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats()); + ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats()); - zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true); - zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c2", true); + zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true); + zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c2", true); - ClusterState state = reader.getClusterState(); + ClusterState state = reader.getClusterState(); - // create collection 2 with stateFormat = 2 - ZkWriteCommand c2 = new ZkWriteCommand("c2", - new DocCollection("c2", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.getCollectionPath("c2"))); - state = writer.enqueueUpdate(reader.getClusterState(), c2, null); - assertFalse(writer.hasPendingUpdates()); // first write is flushed immediately + // create collection 2 with stateFormat = 2 + ZkWriteCommand c2 = new ZkWriteCommand("c2", + new DocCollection("c2", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.getCollectionPath("c2"))); + state = writer.enqueueUpdate(reader.getClusterState(), c2, null); + assertFalse(writer.hasPendingUpdates()); // first write is flushed immediately - int sharedClusterStateVersion = state.getZkClusterStateVersion(); - int stateFormat2Version = state.getCollection("c2").getZNodeVersion(); + int sharedClusterStateVersion = state.getZkClusterStateVersion(); + int stateFormat2Version = state.getCollection("c2").getZNodeVersion(); - // Simulate an external modification to /collections/c2/state.json - byte[] data = zkClient.getData(ZkStateReader.getCollectionPath("c2"), null, null, true); - zkClient.setData(ZkStateReader.getCollectionPath("c2"), data, true); + // Simulate an external modification to /collections/c2/state.json + byte[] data = zkClient.getData(ZkStateReader.getCollectionPath("c2"), null, null, true); + zkClient.setData(ZkStateReader.getCollectionPath("c2"), data, true); - // get the most up-to-date state - reader.forceUpdateCollection("c2"); - state = reader.getClusterState(); - log.info("Cluster state: {}", state); - assertTrue(state.hasCollection("c2")); - assertEquals(sharedClusterStateVersion, (int) state.getZkClusterStateVersion()); - assertEquals(stateFormat2Version + 1, state.getCollection("c2").getZNodeVersion()); + // get the most up-to-date state + reader.forceUpdateCollection("c2"); + state = reader.getClusterState(); + log.info("Cluster state: {}", state); + assertTrue(state.hasCollection("c2")); + assertEquals(sharedClusterStateVersion, (int) state.getZkClusterStateVersion()); + assertEquals(stateFormat2Version + 1, state.getCollection("c2").getZNodeVersion()); - // enqueue an update to stateFormat2 collection such that update is pending - state = writer.enqueueUpdate(state, c2, null); - assertTrue(writer.hasPendingUpdates()); + // enqueue an update to stateFormat2 collection such that update is pending + state = writer.enqueueUpdate(state, c2, null); + assertTrue(writer.hasPendingUpdates()); - // get the most up-to-date state - reader.forceUpdateCollection("c2"); - state = reader.getClusterState(); + // get the most up-to-date state + reader.forceUpdateCollection("c2"); + state = reader.getClusterState(); - // enqueue a stateFormat=1 collection which should cause a flush - ZkWriteCommand c1 = new ZkWriteCommand("c1", - new DocCollection("c1", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.CLUSTER_STATE)); + // enqueue a stateFormat=1 collection which should cause a flush + ZkWriteCommand c1 = new ZkWriteCommand("c1", + new DocCollection("c1", new HashMap(), new HashMap(), DocRouter.DEFAULT, 0, ZkStateReader.CLUSTER_STATE)); - try { - writer.enqueueUpdate(state, c1, null); - fail("Enqueue should not have succeeded"); - } catch (KeeperException.BadVersionException bve) { - // expected - } + try { + writer.enqueueUpdate(state, c1, null); + fail("Enqueue should not have succeeded"); + } catch (KeeperException.BadVersionException bve) { + // expected + } - try { - writer.enqueueUpdate(reader.getClusterState(), c2, null); - fail("enqueueUpdate after BadVersionException should not have succeeded"); - } catch (IllegalStateException e) { - // expected - } + try { + writer.enqueueUpdate(reader.getClusterState(), c2, null); + fail("enqueueUpdate after BadVersionException should not have succeeded"); + } catch (IllegalStateException e) { + // expected + } - try { - writer.writePendingUpdates(); - fail("writePendingUpdates after BadVersionException should not have succeeded"); - } catch (IllegalStateException e) { - // expected + try { + writer.writePendingUpdates(); + fail("writePendingUpdates after BadVersionException should not have succeeded"); + } catch (IllegalStateException e) { + // expected + } } } finally { IOUtils.close(zkClient); diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java index 41e32dd697f..c182495e91a 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java +++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java @@ -22,6 +22,7 @@ import java.io.StringReader; import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -35,8 +36,12 @@ import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; +import org.apache.solr.handler.DumpRequestHandler; import org.apache.solr.handler.TestBlobHandler; import org.apache.solr.handler.TestSolrConfigHandlerConcurrent; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrCache; import org.apache.solr.util.RestTestBase; import org.apache.solr.util.RestTestHarness; import org.eclipse.jetty.servlet.ServletHolder; @@ -449,7 +454,56 @@ public class TestSolrConfigHandler extends RestTestBase { assertEquals(2, initArgs.size()); assertTrue(((Map)initArgs.get(0)).containsKey("suggester")); assertTrue(((Map)initArgs.get(1)).containsKey("suggester")); - System.out.println(map); + + payload = "{\n" + + "'add-requesthandler' : { 'name' : '/dump101', 'class': " + + "'" + CacheTest.class.getName() + "' " + + ", 'startup' : 'lazy'}\n" + + "}"; + runConfigCommand(writeHarness, "/config?wt=json", payload); + + testForResponseElement(writeHarness, + testServerBaseUrl, + "/config/overlay?wt=json", + cloudSolrClient, + Arrays.asList("overlay", "requestHandler", "/dump101", "startup"), + "lazy", + 10); + + payload = "{\n" + + "'add-cache' : {name:'lfuCacheDecayFalse', class:'solr.search.LFUCache', size:10 ,initialSize:9 , timeDecay:false }," + + "'add-cache' : {name: 'perSegFilter', class: 'solr.search.LRUCache', size:10, initialSize:0 , autowarmCount:10}}"; + runConfigCommand(writeHarness, "/config?wt=json", payload); + + map = testForResponseElement(writeHarness, + testServerBaseUrl, + "/config/overlay?wt=json", + cloudSolrClient, + Arrays.asList("overlay", "cache", "lfuCacheDecayFalse", "class"), + "solr.search.LFUCache", + 10); + assertEquals("solr.search.LRUCache",getObjectByPath(map, true, ImmutableList.of("overlay", "cache", "perSegFilter", "class"))); + + map = getRespMap("/dump101?cacheNames=lfuCacheDecayFalse&cacheNames=perSegFilter&wt=json", writeHarness); + assertEquals("Actual output "+ Utils.toJSONString(map), "org.apache.solr.search.LRUCache",getObjectByPath(map, true, ImmutableList.of( "caches", "perSegFilter"))); + assertEquals("Actual output "+ Utils.toJSONString(map), "org.apache.solr.search.LFUCache",getObjectByPath(map, true, ImmutableList.of( "caches", "lfuCacheDecayFalse"))); + + } + + public static class CacheTest extends DumpRequestHandler { + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { + super.handleRequestBody(req, rsp); + String[] caches = req.getParams().getParams("cacheNames"); + if(caches != null && caches.length>0){ + HashMap m = new HashMap(); + rsp.add("caches", m); + for (String c : caches) { + SolrCache cache = req.getSearcher().getCache(c); + if(cache != null) m.put(c, cache.getClass().getName()); + } + } + } } public static Map testForResponseElement(RestTestHarness harness, diff --git a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java new file mode 100644 index 00000000000..aacac5242cb --- /dev/null +++ b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java @@ -0,0 +1,419 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core.snapshots; + +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexCommit; +import org.apache.lucene.index.IndexNotFoundException; +import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.util.LuceneTestCase.Slow; +import org.apache.lucene.util.TestUtil; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest.CreateSnapshot; +import org.apache.solr.client.solrj.request.CoreAdminRequest.DeleteSnapshot; +import org.apache.solr.client.solrj.request.CoreAdminRequest.ListSnapshots; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData; +import org.apache.solr.handler.BackupRestoreUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP; + +@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test +@Slow +public class TestSolrCoreSnapshots extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static long docsSeed; // see indexDocs() + + @BeforeClass + public static void setupClass() throws Exception { + useFactory("solr.StandardDirectoryFactory"); + configureCluster(1)// nodes + .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .configure(); + + docsSeed = random().nextLong(); + } + + @AfterClass + public static void teardownClass() throws Exception { + System.clearProperty("test.build.data"); + System.clearProperty("test.cache.data"); + } + + @Test + public void testBackupRestore() throws Exception { + CloudSolrClient solrClient = cluster.getSolrClient(); + String collectionName = "SolrCoreSnapshots"; + CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1); + create.process(solrClient); + + String location = createTempDir().toFile().getAbsolutePath(); + int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed); + + DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName); + assertEquals(1, collectionState.getActiveSlices().size()); + Slice shard = collectionState.getActiveSlices().iterator().next(); + assertEquals(1, shard.getReplicas().size()); + Replica replica = shard.getReplicas().iterator().next(); + + String replicaBaseUrl = replica.getStr(BASE_URL_PROP); + String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP); + String backupName = TestUtil.randomSimpleString(random(), 1, 5); + String commitName = TestUtil.randomSimpleString(random(), 1, 5); + String duplicateName = commitName.concat("_duplicate"); + + try ( + SolrClient adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + SolrClient masterClient = getHttpSolrClient(replica.getCoreUrl())) { + + SnapshotMetaData metaData = createSnapshot(adminClient, coreName, commitName); + // Create another snapshot referring to the same index commit to verify the + // reference counting implementation during snapshot deletion. + SnapshotMetaData duplicateCommit = createSnapshot(adminClient, coreName, duplicateName); + + assertEquals (metaData.getIndexDirPath(), duplicateCommit.getIndexDirPath()); + assertEquals (metaData.getGenerationNumber(), duplicateCommit.getGenerationNumber()); + + // Delete all documents + masterClient.deleteByQuery("*:*"); + masterClient.commit(); + BackupRestoreUtils.verifyDocs(0, cluster.getSolrClient(), collectionName); + + // Verify that the index directory contains at least 2 index commits - one referred by the snapshots + // and the other containing document deletions. + { + List commits = listCommits(metaData.getIndexDirPath()); + assertTrue(2 <= commits.size()); + } + + // Backup the earlier created snapshot. + { + Map params = new HashMap<>(); + params.put("name", backupName); + params.put("commitName", commitName); + params.put("location", location); + BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params); + } + + // Restore the backup + { + Map params = new HashMap<>(); + params.put("name", "snapshot." + backupName); + params.put("location", location); + BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params); + BackupRestoreUtils.verifyDocs(nDocs, cluster.getSolrClient(), collectionName); + } + + // Verify that the old index directory (before restore) contains only those index commits referred by snapshots. + { + List commits = listCommits(metaData.getIndexDirPath()); + assertEquals(1, commits.size()); + assertEquals(metaData.getGenerationNumber(), commits.get(0).getGeneration()); + } + + // Delete first snapshot + deleteSnapshot(adminClient, coreName, commitName); + + // Verify that corresponding index files have NOT been deleted (due to reference counting). + assertFalse(listCommits(metaData.getIndexDirPath()).isEmpty()); + + // Delete second snapshot + deleteSnapshot(adminClient, coreName, duplicateCommit.getName()); + + // Verify that corresponding index files have been deleted. + assertTrue(listCommits(duplicateCommit.getIndexDirPath()).isEmpty()); + } + } + + @Test + public void testHandlingSharedIndexFiles() throws Exception { + CloudSolrClient solrClient = cluster.getSolrClient(); + String collectionName = "SolrCoreSnapshots_IndexFileSharing"; + CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1); + create.process(solrClient); + + int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed); + DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName); + assertEquals(1, collectionState.getActiveSlices().size()); + Slice shard = collectionState.getActiveSlices().iterator().next(); + assertEquals(1, shard.getReplicas().size()); + Replica replica = shard.getReplicas().iterator().next(); + + String replicaBaseUrl = replica.getStr(BASE_URL_PROP); + String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP); + String backupName = TestUtil.randomSimpleString(random(), 1, 5); + String location = createTempDir().toFile().getAbsolutePath(); + + try ( + SolrClient adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + SolrClient masterClient = getHttpSolrClient(replica.getCoreUrl())) { + + int numTests = TestUtil.nextInt(random(), 2, 5); + List snapshots = new ArrayList<>(numTests); + + // Create multiple commits and create a snapshot per commit. + // This should result in Lucene reusing some of the segments for later index commits. + for (int attempt=0; attempt 0) { + //Delete a few docs + int numDeletes = TestUtil.nextInt(random(), 1, nDocs); + for(int i=0; i params = new HashMap<>(); + params.put("name", backupName); + params.put("commitName", snapshots.get(0).getName()); + params.put("location", location); + BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params); + } + + // Restore the backup. The purpose of the restore operation is to change the *current* index directory. + // This is required since we delegate the file deletion to underlying IndexDeletionPolicy in case of + // *current* index directory. Hence for the purpose of this test, we want to ensure that the created + // snapshots are NOT in the *current* index directory. + { + Map params = new HashMap<>(); + params.put("name", "snapshot." + backupName); + params.put("location", location); + BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params); + } + + { + SnapshotMetaData snapshotMetaData = snapshots.get(0); + + List commits = listCommits(snapshotMetaData.getIndexDirPath()); + // Check if number of index commits are > 0 to ensure index file sharing. + assertTrue(commits.size() > 0); + Map refCounts = SolrSnapshotManager.buildRefCounts(snapshots, commits); + + Optional ic = commits.stream() + .filter(entry -> entry.getGeneration() == snapshotMetaData.getGenerationNumber()) + .findFirst(); + assertTrue(ic.isPresent()); + Collection nonSharedFiles = new ArrayList<>(); + Collection sharedFiles = new ArrayList<>(); + for (String fileName : ic.get().getFileNames()) { + if (refCounts.getOrDefault(fileName, 0) > 1) { + sharedFiles.add(fileName); + } else { + nonSharedFiles.add(fileName); + } + } + + // Delete snapshot + deleteSnapshot(adminClient, coreName, snapshotMetaData.getName()); + + // Verify that the shared files are not deleted. + for (String fileName : sharedFiles) { + Path path = Paths.get(snapshotMetaData.getIndexDirPath(), fileName); + assertTrue(path + " should exist.", Files.exists(path)); + } + + // Verify that the non-shared files are deleted. + for (String fileName : nonSharedFiles) { + Path path = Paths.get(snapshotMetaData.getIndexDirPath(), fileName); + assertFalse(path + " should not exist.", Files.exists(path)); + } + } + } + } + + @Test + public void testIndexOptimization() throws Exception { + CloudSolrClient solrClient = cluster.getSolrClient(); + String collectionName = "SolrCoreSnapshots_IndexOptimization"; + CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1); + create.process(solrClient); + + int nDocs = BackupRestoreUtils.indexDocs(cluster.getSolrClient(), collectionName, docsSeed); + + DocCollection collectionState = solrClient.getZkStateReader().getClusterState().getCollection(collectionName); + assertEquals(1, collectionState.getActiveSlices().size()); + Slice shard = collectionState.getActiveSlices().iterator().next(); + assertEquals(1, shard.getReplicas().size()); + Replica replica = shard.getReplicas().iterator().next(); + + String coreName = replica.getStr(ZkStateReader.CORE_NAME_PROP); + String commitName = TestUtil.randomSimpleString(random(), 1, 5); + + try ( + SolrClient adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString()); + SolrClient masterClient = getHttpSolrClient(replica.getCoreUrl())) { + + SnapshotMetaData metaData = createSnapshot(adminClient, coreName, commitName); + + int numTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1; + for (int attempt=0; attempt 0) { + //Delete a few docs + int numDeletes = TestUtil.nextInt(random(), 1, nDocs); + for(int i=0; i result = new ArrayList<>(apiResult.size()); + for(int i = 0 ; i < apiResult.size(); i++) { + String commitName = apiResult.getName(i); + String indexDirPath = (String)((NamedList)apiResult.get(commitName)).get("indexDirPath"); + long genNumber = Long.valueOf((String)((NamedList)apiResult.get(commitName)).get("generation")); + result.add(new SnapshotMetaData(commitName, indexDirPath, genNumber)); + } + return result; + } + + private List listCommits(String directory) throws Exception { + SimpleFSDirectory dir = new SimpleFSDirectory(Paths.get(directory)); + try { + return DirectoryReader.listCommits(dir); + } catch (IndexNotFoundException ex) { + // This can happen when the delete snapshot functionality cleans up the index files (when the directory + // storing these files is not the *current* index directory). + return Collections.emptyList(); + } + } +} \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/handler/BackupRestoreUtils.java b/solr/core/src/test/org/apache/solr/handler/BackupRestoreUtils.java index e2f4304ac82..34509cf8b3e 100644 --- a/solr/core/src/test/org/apache/solr/handler/BackupRestoreUtils.java +++ b/solr/core/src/test/org/apache/solr/handler/BackupRestoreUtils.java @@ -18,11 +18,15 @@ package org.apache.solr.handler; import java.io.IOException; +import java.io.InputStream; import java.lang.invoke.MethodHandles; +import java.net.URL; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Random; +import org.apache.commons.io.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.apache.solr.client.solrj.SolrClient; @@ -64,4 +68,37 @@ public class BackupRestoreUtils extends LuceneTestCase { assertEquals(0, response.getStatus()); assertEquals(nDocs, response.getResults().getNumFound()); } + + public static void runCoreAdminCommand(String baseUrl, String coreName, String action, Map params) throws IOException { + StringBuilder builder = new StringBuilder(); + builder.append(baseUrl); + builder.append("/admin/cores?action="); + builder.append(action); + builder.append("&core="); + builder.append(coreName); + for (Map.Entry p : params.entrySet()) { + builder.append("&"); + builder.append(p.getKey()); + builder.append("="); + builder.append(p.getValue()); + } + String masterUrl = builder.toString(); + executeHttpRequest(masterUrl); + } + + public static void runReplicationHandlerCommand(String baseUrl, String coreName, String action, String repoName, String backupName) throws IOException { + String masterUrl = baseUrl + "/" + coreName + ReplicationHandler.PATH + "?command=" + action + "&repository="+repoName+"&name="+backupName; + executeHttpRequest(masterUrl); + } + + static void executeHttpRequest(String requestUrl) throws IOException { + InputStream stream = null; + try { + URL url = new URL(requestUrl); + stream = url.openStream(); + stream.close(); + } finally { + IOUtils.closeQuietly(stream); + } + } } diff --git a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java index a84042833ab..4e8d4ccd584 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java +++ b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java @@ -18,11 +18,11 @@ package org.apache.solr.handler; import java.io.IOException; -import java.io.InputStream; import java.lang.invoke.MethodHandles; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; +import java.util.HashMap; +import java.util.Map; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.commons.io.IOUtils; @@ -44,6 +44,7 @@ import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction; import org.apache.solr.util.BadHdfsThreadsFilter; import org.junit.AfterClass; @@ -176,16 +177,19 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase { try (SolrClient masterClient = getHttpSolrClient(replicaBaseUrl)) { // Create a backup. if (testViaReplicationHandler) { - log.info("Running Backup/restore via replication handler"); - runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName); + log.info("Running Backup via replication handler"); + BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName); CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, coreName, null); while (!checkBackupStatus.success) { checkBackupStatus.fetchStatus(); Thread.sleep(1000); } } else { - log.info("Running Backup/restore via core admin api"); - runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), "hdfs", backupName); + log.info("Running Backup via core admin api"); + Map params = new HashMap<>(); + params.put("name", backupName); + params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs"); + BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params); } int numRestoreTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1; @@ -214,38 +218,22 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase { } // Snapshooter prefixes "snapshot." to the backup name. if (testViaReplicationHandler) { + log.info("Running Restore via replication handler"); // Snapshooter prefixes "snapshot." to the backup name. - runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_RESTORE, "hdfs", backupName); + BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_RESTORE, "hdfs", backupName); while (!TestRestoreCore.fetchRestoreStatus(baseUrl, coreName)) { Thread.sleep(1000); } } else { - runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), "hdfs", "snapshot." + backupName); + log.info("Running Restore via core admin api"); + Map params = new HashMap<>(); + params.put("name", "snapshot." + backupName); + params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs"); + BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params); } //See if restore was successful by checking if all the docs are present again BackupRestoreUtils.verifyDocs(nDocs, masterClient, coreName); } } } - - static void runCoreAdminCommand(String baseUrl, String coreName, String action, String repoName, String backupName) throws IOException { - String masterUrl = baseUrl + "/admin/cores?action=" + action + "&core="+coreName+"&repository="+repoName+"&name="+backupName; - executeHttpRequest(masterUrl); - } - - static void runReplicationHandlerCommand(String baseUrl, String coreName, String action, String repoName, String backupName) throws IOException { - String masterUrl = baseUrl + "/" + coreName + ReplicationHandler.PATH + "?command=" + action + "&repository="+repoName+"&name="+backupName; - executeHttpRequest(masterUrl); - } - - static void executeHttpRequest(String requestUrl) throws IOException { - InputStream stream = null; - try { - URL url = new URL(requestUrl); - stream = url.openStream(); - stream.close(); - } finally { - IOUtils.closeQuietly(stream); - } - } } diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java index b9bcf7b254b..08c462bc70b 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java @@ -20,6 +20,7 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -66,6 +67,7 @@ import org.apache.solr.core.CachingDirectoryFactory; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.core.StandardDirectoryFactory; +import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.util.FileUtils; import org.junit.After; import org.junit.Before; @@ -896,8 +898,8 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { CachingDirectoryFactory dirFactory = (CachingDirectoryFactory) core.getDirectoryFactory(); synchronized (dirFactory) { Set livePaths = dirFactory.getLivePaths(); - // one for data, one for hte index under data - assertEquals(livePaths.toString(), 2, livePaths.size()); + // one for data, one for hte index under data and one for the snapshot metadata. + assertEquals(livePaths.toString(), 3, livePaths.size()); // :TODO: assert that one of the paths is a subpath of hte other } if (dirFactory instanceof StandardDirectoryFactory) { @@ -908,14 +910,14 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { } private int indexDirCount(String ddir) { - String[] list = new File(ddir).list(); - int cnt = 0; - for (String file : list) { - if (!file.endsWith(".properties")) { - cnt++; + String[] list = new File(ddir).list(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + File f = new File(dir, name); + return f.isDirectory() && !SolrSnapshotMetaDataManager.SNAPSHOT_METADATA_DIR.equals(name); } - } - return cnt; + }); + return list.length; } private void pullFromMasterToSlave() throws MalformedURLException, diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java index 2a894732ea3..86c7ee8248f 100644 --- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java +++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java @@ -15,6 +15,11 @@ * limitations under the License. */ package org.apache.solr.search; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryUtils; import org.apache.solr.SolrTestCaseJ4; @@ -24,10 +29,6 @@ import org.apache.solr.response.SolrQueryResponse; import org.junit.AfterClass; import org.junit.BeforeClass; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - /** @@ -174,6 +175,24 @@ public class QueryEqualityTest extends SolrTestCaseJ4 { } } + public void testTlogitQuery() throws Exception { + SolrQueryRequest req = req("q", "*:*", "feature", "f", "terms","a,b,c", "weights", "100,200,300", "idfs","1,5,7","iteration","1", "outcome","a","positiveLabel","1"); + try { + assertQueryEquals("tlogit", req, "{!tlogit}"); + } finally { + req.close(); + } + } + + public void testIGainQuery() throws Exception { + SolrQueryRequest req = req("q", "*:*", "outcome", "b", "positiveLabel", "1", "field", "x", "numTerms","200"); + try { + assertQueryEquals("igain", req, "{!igain}"); + } finally { + req.close(); + } + } + public void testQuerySwitch() throws Exception { SolrQueryRequest req = req("myXXX", "XXX", "myField", "foo_s", @@ -1075,4 +1094,22 @@ public class QueryEqualityTest extends SolrTestCaseJ4 { // assertFuncEquals("agg_multistat(foo_i)", "agg_multistat(foo_i)"); } + public void testCompares() throws Exception { + assertFuncEquals("gt(foo_i,2)", "gt(foo_i, 2)"); + assertFuncEquals("gt(foo_i,2)", "gt(foo_i,2)"); + assertFuncEquals("lt(foo_i,2)", "lt(foo_i,2)"); + assertFuncEquals("lte(foo_i,2)", "lte(foo_i,2)"); + assertFuncEquals("gte(foo_i,2)", "gte(foo_i,2)"); + assertFuncEquals("eq(foo_i,2)", "eq(foo_i,2)"); + + boolean equals = false; + try { + assertFuncEquals("eq(foo_i,2)", "lt(foo_i,2)"); + equals = true; + } catch (AssertionError e) { + //expected + } + assertFalse(equals); + } + } diff --git a/solr/core/src/test/org/apache/solr/search/TestOverriddenPrefixQueryForCustomFieldType.java b/solr/core/src/test/org/apache/solr/search/TestOverriddenPrefixQueryForCustomFieldType.java index 42b0d37cf3c..15012ccfa1b 100644 --- a/solr/core/src/test/org/apache/solr/search/TestOverriddenPrefixQueryForCustomFieldType.java +++ b/solr/core/src/test/org/apache/solr/search/TestOverriddenPrefixQueryForCustomFieldType.java @@ -133,7 +133,7 @@ public class TestOverriddenPrefixQueryForCustomFieldType extends SolrTestCaseJ4 SolrQueryResponse rsp = new SolrQueryResponse(); SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); for (int i = 0; i < inputs.length; i++) { - queries[i] = (QParser.getParser(inputs[i], null, req).getQuery()); + queries[i] = (QParser.getParser(inputs[i], req).getQuery()); } } finally { SolrRequestInfo.clearRequestInfo(); diff --git a/solr/core/src/test/org/apache/solr/search/TestPseudoReturnFields.java b/solr/core/src/test/org/apache/solr/search/TestPseudoReturnFields.java index 87f3d89c021..0a987345ed2 100644 --- a/solr/core/src/test/org/apache/solr/search/TestPseudoReturnFields.java +++ b/solr/core/src/test/org/apache/solr/search/TestPseudoReturnFields.java @@ -30,6 +30,7 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.commons.lang.StringUtils; +import org.junit.Before; import org.junit.BeforeClass; @@ -61,9 +62,14 @@ public class TestPseudoReturnFields extends SolrTestCaseJ4 { assertU(adoc("id", "46", "val_i", "3", "ssto", "X", "subject", "ggg")); assertU(commit()); - // uncommitted doc in transaction log + } + + @Before + private void addUncommittedDoc99() throws Exception { + // uncommitted doc in transaction log at start of every test + // Even if an RTG causes ulog to re-open realtime searcher, next test method + // will get another copy of doc 99 in the ulog assertU(adoc("id", "99", "val_i", "1", "ssto", "X", "subject", "uncommitted")); - } public void testMultiValued() throws Exception { @@ -140,8 +146,19 @@ public class TestPseudoReturnFields extends SolrTestCaseJ4 { ); } } - - + } + + public void testFilterAndOneRealFieldRTG() throws Exception { + // shouldn't matter if we use RTG (committed or otherwise) + // only one of these docs should match... + assertQ("RTG w/ 2 ids & fq that only matches 1 uncommitted doc", + req("qt","/get","ids","42,99", "wt","xml","fl","id,val_i", + "fq","{!field f='subject' v=$my_var}","my_var","uncommitted") + ,"//result[@numFound='1']" + ,"//result/doc/str[@name='id'][.='99']" + ,"//result/doc/int[@name='val_i'][.='1']" + ,"//result/doc[count(*)=2]" + ); } public void testScoreAndAllRealFields() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java b/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java index 42d13ade879..8ad807cd308 100644 --- a/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java +++ b/solr/core/src/test/org/apache/solr/search/TestSearchPerf.java @@ -198,10 +198,10 @@ public class TestSearchPerf extends AbstractSolrTestCase { String u=t((int)(indexSize*10*fractionCovered)); SolrQueryRequest req = lrf.makeRequest(); - QParser parser = QParser.getParser("foomany_s:[" + l + " TO " + u + "]", null, req); + QParser parser = QParser.getParser("foomany_s:[" + l + " TO " + u + "]", req); Query range = parser.getQuery(); - QParser parser2 = QParser.getParser("{!frange l="+l+" u="+u+"}foomany_s", null, req); + QParser parser2 = QParser.getParser("{!frange l="+l+" u="+u+"}foomany_s", req); Query frange = parser2.getQuery(); req.close(); @@ -224,13 +224,13 @@ public class TestSearchPerf extends AbstractSolrTestCase { SolrQueryRequest req = lrf.makeRequest(); - QParser parser = QParser.getParser("foomany_s:[" + l + " TO " + u + "]", null, req); + QParser parser = QParser.getParser("foomany_s:[" + l + " TO " + u + "]", req); Query rangeQ = parser.getQuery(); List filters = new ArrayList<>(); filters.add(rangeQ); req.close(); - parser = QParser.getParser("{!dismax qf=t10_100_ws pf=t10_100_ws ps=20}"+ t(0) + ' ' + t(1) + ' ' + t(2), null, req); + parser = QParser.getParser("{!dismax qf=t10_100_ws pf=t10_100_ws ps=20}"+ t(0) + ' ' + t(1) + ' ' + t(2), req); Query q= parser.getQuery(); // SolrIndexSearcher searcher = req.getSearcher(); diff --git a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java index 5ee1f7f984b..0b9c0f0c181 100644 --- a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java +++ b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java @@ -169,13 +169,13 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 { public void testCSQ() throws Exception { SolrQueryRequest req = req(); - QParser qParser = QParser.getParser("text:x^=3", "lucene", req); + QParser qParser = QParser.getParser("text:x^=3", req); Query q = qParser.getQuery(); assertTrue(q instanceof BoostQuery); assertTrue(((BoostQuery) q).getQuery() instanceof ConstantScoreQuery); assertEquals(3.0, ((BoostQuery) q).getBoost(), 0.0f); - qParser = QParser.getParser("(text:x text:y)^=-3", "lucene", req); + qParser = QParser.getParser("(text:x text:y)^=-3", req); q = qParser.getQuery(); assertTrue(q instanceof BoostQuery); assertTrue(((BoostQuery) q).getQuery() instanceof ConstantScoreQuery); diff --git a/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java b/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java index 4c7c4c1a7f5..ff9ffffcdfa 100644 --- a/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java +++ b/solr/core/src/test/org/apache/solr/search/TestStandardQParsers.java @@ -79,4 +79,13 @@ public class TestStandardQParsers extends LuceneTestCase { } + /** + * Test that "lucene" is the default query parser. + */ + @Test + public void testDefaultQType() throws Exception { + assertEquals(LuceneQParserPlugin.NAME, QParserPlugin.DEFAULT_QTYPE); + assertEquals("lucene", LuceneQParserPlugin.NAME); + } + } diff --git a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java index f94a9ee1739..8c65b588559 100644 --- a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java +++ b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java @@ -98,7 +98,7 @@ public class TestFunctionQuery extends SolrTestCaseJ4 { return sb.toString(); } - void singleTest(String field, String funcTemplate, List args, float... results) { + protected void singleTest(String field, String funcTemplate, List args, float... results) { String parseableQuery = func(field, funcTemplate); List nargs = new ArrayList<>(Arrays.asList("q", parseableQuery @@ -793,4 +793,69 @@ public class TestFunctionQuery extends SolrTestCaseJ4 { } } -} + @Test + public void testNumericComparisons() throws Exception { + assertU(adoc("id", "1", "age_i", "35")); + assertU(adoc("id", "2", "age_i", "25")); + assertU(commit()); + + // test weighting of functions + assertJQ(req("q", "id:1", "fl", "a:gt(age_i,30),b:lt(age_i,30)") + , "/response/docs/[0]=={'a':true,'b':false}"); + + assertJQ(req("q", "id:1", "fl", "a:exists(gt(foo_i,30))") + , "/response/docs/[0]=={'a':false}"); + + singleTest("age_i", "if(gt(age_i,30),5,2)", + /*id*/1, /*score*/5, + /*id*/2, /*score*/2); + + singleTest("age_i", "if(lt(age_i,30),5,2)", + /*id*/1, /*score*/2, + /*id*/2, /*score*/5); + + singleTest("age_i", "if(lt(age_i,34.5),5,2)", + /*id*/1, /*score*/2, + /*id*/2, /*score*/5); + + singleTest("age_i", "if(lte(age_i,35),5,2)", + /*id*/1, /*score*/5, + /*id*/2, /*score*/5); + + singleTest("age_i", "if(gte(age_i,25),5,2)", + /*id*/1, /*score*/5, + /*id*/2, /*score*/5); + + singleTest("age_i", "if(lte(age_i,25),5,2)", + /*id*/1, /*score*/2, + /*id*/2, /*score*/5); + + singleTest("age_i", "if(gte(age_i,35),5,2)", + /*id*/1, /*score*/5, + /*id*/2, /*score*/2); + + + singleTest("age_i", "if(eq(age_i,30),5,2)", + /*id*/1, /*score*/2, + /*id*/2, /*score*/2); + + singleTest("age_i", "if(eq(age_i,35),5,2)", + /*id*/1, /*score*/5, + /*id*/2, /*score*/2); + } + + public void testLongComparisons() { + assertU(adoc("id", "1", "number_of_atoms_in_universe_l", Long.toString(Long.MAX_VALUE))); + assertU(adoc("id", "2", "number_of_atoms_in_universe_l", Long.toString(Long.MAX_VALUE - 1))); + assertU(commit()); + + singleTest("number_of_atoms_in_universe_l", "if(gt(number_of_atoms_in_universe_l," + Long.toString(Long.MAX_VALUE - 1) + "),5,2)", + /*id*/1, /*score*/5, + /*id*/2, /*score*/2); + + singleTest("number_of_atoms_in_universe_l", "if(lt(number_of_atoms_in_universe_l," + Long.toString(Long.MAX_VALUE) + "),5,2)", + /*id*/2, /*score*/5, + /*id*/1, /*score*/2); + } + + } diff --git a/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java index 52b139c855b..3b23be8c9a5 100644 --- a/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java +++ b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPNoScore.java @@ -168,7 +168,7 @@ public class TestScoreJoinQPNoScore extends SolrTestCaseJ4 { SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); { - final Query query = QParser.getParser(req.getParams().get("q"), null, req).getQuery(); + final Query query = QParser.getParser(req.getParams().get("q"), req).getQuery(); final Query rewrittenQuery = query.rewrite(req.getSearcher().getIndexReader()); assertTrue( rewrittenQuery+" should be Lucene's", @@ -178,7 +178,7 @@ public class TestScoreJoinQPNoScore extends SolrTestCaseJ4 { { final Query query = QParser.getParser( "{!join from=dept_id_s to=dept_ss}text_t:develop" - , null, req).getQuery(); + , req).getQuery(); final Query rewrittenQuery = query.rewrite(req.getSearcher().getIndexReader()); assertEquals(rewrittenQuery+" is expected to be from Solr", JoinQParserPlugin.class.getPackage().getName(), diff --git a/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java index a674d503ec2..51039d64127 100644 --- a/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java +++ b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java @@ -187,7 +187,7 @@ public class TestScoreJoinQPScore extends SolrTestCaseJ4 { final SolrQueryRequest req = req("q", "{!join from=movieId_s to=id score=" + score + " b=200}title:movie", "fl", "id,score", "omitHeader", "true"); SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, new SolrQueryResponse())); - final Query luceneQ = QParser.getParser(req.getParams().get("q"), null, req).getQuery().rewrite(req.getSearcher().getLeafReader()); + final Query luceneQ = QParser.getParser(req.getParams().get("q"), req).getQuery().rewrite(req.getSearcher().getLeafReader()); assertTrue(luceneQ instanceof BoostQuery); float boost = ((BoostQuery) luceneQ).getBoost(); assertEquals("" + luceneQ, Float.floatToIntBits(200), Float.floatToIntBits(boost)); diff --git a/solr/core/src/test/org/apache/solr/security/HttpParamDelegationTokenPlugin.java b/solr/core/src/test/org/apache/solr/security/HttpParamDelegationTokenPlugin.java new file mode 100644 index 00000000000..7a4f69fa903 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/security/HttpParamDelegationTokenPlugin.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.security.Principal; +import java.util.Enumeration; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.security.authentication.server.AuthenticationHandler; +import org.apache.hadoop.security.authentication.server.AuthenticationToken; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; + +import org.apache.http.HttpException; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.http.protocol.HttpContext; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.request.SolrRequestInfo; + +/** + * AuthenticationHandler that supports delegation tokens and simple + * authentication via the "user" http parameter + */ +public class HttpParamDelegationTokenPlugin extends KerberosPlugin { + public static final String USER_PARAM = "user"; // http parameter for user authentication + public static final String REMOTE_HOST_PARAM = "remoteHost"; // http parameter for indicating remote host + public static final String REMOTE_ADDRESS_PARAM = "remoteAddress"; // http parameter for indicating remote address + public static final String INTERNAL_REQUEST_HEADER = "internalRequest"; // http header for indicating internal request + + boolean isSolrThread() { + return ExecutorUtil.isSolrServerThread(); + } + + private final HttpRequestInterceptor interceptor = new HttpRequestInterceptor() { + @Override + public void process(HttpRequest httpRequest, HttpContext httpContext) throws HttpException, IOException { + SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo(); + String usr; + if (reqInfo != null) { + Principal principal = reqInfo.getReq().getUserPrincipal(); + if (principal == null) { + //this had a request but not authenticated + //so we don't not need to set a principal + return; + } else { + usr = principal.getName(); + } + } else { + if (!isSolrThread()) { + //if this is not running inside a Solr threadpool (as in testcases) + // then no need to add any header + return; + } + //this request seems to be originated from Solr itself + usr = "$"; //special name to denote the user is the node itself + } + httpRequest.setHeader(INTERNAL_REQUEST_HEADER, usr); + } + }; + + public HttpParamDelegationTokenPlugin(CoreContainer coreContainer) { + super(coreContainer); + } + + @Override + public void init(Map pluginConfig) { + try { + final FilterConfig initConf = getInitFilterConfig(pluginConfig, true); + + FilterConfig conf = new FilterConfig() { + @Override + public ServletContext getServletContext() { + return initConf.getServletContext(); + } + + @Override + public Enumeration getInitParameterNames() { + return initConf.getInitParameterNames(); + } + + @Override + public String getInitParameter(String param) { + if (AuthenticationFilter.AUTH_TYPE.equals(param)) { + return HttpParamDelegationTokenAuthenticationHandler.class.getName(); + } + return initConf.getInitParameter(param); + } + + @Override + public String getFilterName() { + return "HttpParamFilter"; + } + }; + Filter kerberosFilter = new HttpParamToRequestFilter(); + kerberosFilter.init(conf); + setKerberosFilter(kerberosFilter); + } catch (ServletException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Error initializing kerberos authentication plugin: "+e); + } + } + + @Override + public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) { + HttpClientUtil.addRequestInterceptor(interceptor); + return super.getHttpClientBuilder(builder); + } + + @Override + public void close() { + HttpClientUtil.removeRequestInterceptor(interceptor); + super.close(); + } + + private static String getHttpParam(HttpServletRequest request, String param) { + List pairs = URLEncodedUtils.parse(request.getQueryString(), Charset.forName("UTF-8")); + for (NameValuePair nvp : pairs) { + if (param.equals(nvp.getName())) { + return nvp.getValue(); + } + } + return null; + } + + public static class HttpParamDelegationTokenAuthenticationHandler extends + DelegationTokenAuthenticationHandler { + + public HttpParamDelegationTokenAuthenticationHandler() { + super(new HttpParamAuthenticationHandler()); + } + + @Override + public void init(Properties config) throws ServletException { + Properties conf = new Properties(); + for (Map.Entry entry : config.entrySet()) { + conf.setProperty((String) entry.getKey(), (String) entry.getValue()); + } + conf.setProperty(TOKEN_KIND, KerberosPlugin.DELEGATION_TOKEN_TYPE_DEFAULT); + super.init(conf); + } + + private static class HttpParamAuthenticationHandler implements AuthenticationHandler { + @Override + public String getType() { + return "dummy"; + } + + @Override + public void init(Properties config) throws ServletException { + } + + @Override + public void destroy() { + } + + @Override + public boolean managementOperation(AuthenticationToken token, + HttpServletRequest request, HttpServletResponse response) + throws IOException, AuthenticationException { + return false; + } + + @Override + public AuthenticationToken authenticate(HttpServletRequest request, + HttpServletResponse response) + throws IOException, AuthenticationException { + AuthenticationToken token = null; + String userName = getHttpParam(request, USER_PARAM); + if (userName == null) { + //check if this is an internal request + userName = request.getHeader(INTERNAL_REQUEST_HEADER); + } + if (userName != null) { + return new AuthenticationToken(userName, userName, "test"); + } else { + response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + response.setHeader("WWW-Authenticate", "dummy"); + } + return token; + } + } + } + + /** + * Filter that converts http params to HttpServletRequest params + */ + private static class HttpParamToRequestFilter extends DelegationTokenKerberosFilter { + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + final HttpServletRequest httpRequest = (HttpServletRequest) request; + final HttpServletRequestWrapper requestWrapper = new HttpServletRequestWrapper(httpRequest) { + @Override + public String getRemoteHost() { + String param = getHttpParam(httpRequest, REMOTE_HOST_PARAM); + return param != null ? param : httpRequest.getRemoteHost(); + } + + @Override + public String getRemoteAddr() { + String param = getHttpParam(httpRequest, REMOTE_ADDRESS_PARAM); + return param != null ? param : httpRequest.getRemoteAddr(); + } + }; + + super.doFilter(requestWrapper, response, chain); + } + + @Override + protected void doFilter(FilterChain filterChain, HttpServletRequest request, + HttpServletResponse response) throws IOException, ServletException { + // remove the filter-specific authentication information, so it doesn't get accidentally forwarded. + List newPairs = new LinkedList(); + List pairs = URLEncodedUtils.parse(request.getQueryString(), Charset.forName("UTF-8")); + for (NameValuePair nvp : pairs) { + if (!USER_PARAM.equals(nvp.getName())) { + newPairs.add(nvp); + } + else { + request.setAttribute(USER_PARAM, nvp.getValue()); + } + } + final String queryStringNoUser = URLEncodedUtils.format(newPairs, StandardCharsets.UTF_8); + HttpServletRequest requestWrapper = new HttpServletRequestWrapper(request) { + @Override + public String getQueryString() { + return queryStringNoUser; + } + }; + super.doFilter(filterChain, requestWrapper, response); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/security/MockAuthenticationPlugin.java b/solr/core/src/test/org/apache/solr/security/MockAuthenticationPlugin.java index e3cf7bdd1b7..3013086d63d 100644 --- a/solr/core/src/test/org/apache/solr/security/MockAuthenticationPlugin.java +++ b/solr/core/src/test/org/apache/solr/security/MockAuthenticationPlugin.java @@ -20,11 +20,16 @@ import javax.servlet.FilterChain; import javax.servlet.ServletException; import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; import java.io.IOException; import java.security.Principal; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Predicate; +import org.apache.http.auth.BasicUserPrincipal; + public class MockAuthenticationPlugin extends AuthenticationPlugin { static Predicate predicate; @@ -33,7 +38,7 @@ public class MockAuthenticationPlugin extends AuthenticationPlugin { } @Override - public void doAuthenticate(ServletRequest request, ServletResponse response, FilterChain filterChain) throws IOException, ServletException { + public boolean doAuthenticate(ServletRequest request, ServletResponse response, FilterChain filterChain) throws IOException, ServletException { String user = null; if (predicate != null) { if (predicate.test(request)) { @@ -41,9 +46,32 @@ public class MockAuthenticationPlugin extends AuthenticationPlugin { request.removeAttribute(Principal.class.getName()); } } - forward(user, request, response, filterChain); + + final FilterChain ffc = filterChain; + final AtomicBoolean requestContinues = new AtomicBoolean(false); + forward(user, request, response, new FilterChain() { + @Override + public void doFilter(ServletRequest req, ServletResponse res) throws IOException, ServletException { + ffc.doFilter(req, res); + requestContinues.set(true); + } + }); + return requestContinues.get(); } + protected void forward(String user, ServletRequest req, ServletResponse rsp, + FilterChain chain) throws IOException, ServletException { + if(user != null) { + final Principal p = new BasicUserPrincipal(user); + req = new HttpServletRequestWrapper((HttpServletRequest) req) { + @Override + public Principal getUserPrincipal() { + return p; + } + }; + } + chain.doFilter(req,rsp); + } @Override public void close() throws IOException { diff --git a/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java b/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java index 2bde1186ad6..ca604fed5f8 100644 --- a/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java @@ -160,13 +160,13 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 { SolrQueryRequest ureq = req(); CommitUpdateCommand cmtCmd = new CommitUpdateCommand(ureq, false); cmtCmd.waitSearcher = true; - assertEquals( 1, duh2.addCommands.get() ); - assertEquals( 1, duh2.addCommandsCumulative.get() ); - assertEquals( 0, duh2.commitCommands.get() ); + assertEquals( 1, duh2.addCommands.longValue() ); + assertEquals( 1, duh2.addCommandsCumulative.longValue() ); + assertEquals( 0, duh2.commitCommands.longValue() ); updater.commit(cmtCmd); - assertEquals( 0, duh2.addCommands.get() ); - assertEquals( 1, duh2.addCommandsCumulative.get() ); - assertEquals( 1, duh2.commitCommands.get() ); + assertEquals( 0, duh2.addCommands.longValue() ); + assertEquals( 1, duh2.addCommandsCumulative.longValue() ); + assertEquals( 1, duh2.commitCommands.longValue() ); ureq.close(); assertU(adoc("id","B")); @@ -174,13 +174,13 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 { // rollback "B" ureq = req(); RollbackUpdateCommand rbkCmd = new RollbackUpdateCommand(ureq); - assertEquals( 1, duh2.addCommands.get() ); - assertEquals( 2, duh2.addCommandsCumulative.get() ); - assertEquals( 0, duh2.rollbackCommands.get() ); + assertEquals( 1, duh2.addCommands.longValue() ); + assertEquals( 2, duh2.addCommandsCumulative.longValue() ); + assertEquals( 0, duh2.rollbackCommands.longValue() ); updater.rollback(rbkCmd); - assertEquals( 0, duh2.addCommands.get() ); - assertEquals( 1, duh2.addCommandsCumulative.get() ); - assertEquals( 1, duh2.rollbackCommands.get() ); + assertEquals( 0, duh2.addCommands.longValue() ); + assertEquals( 1, duh2.addCommandsCumulative.longValue() ); + assertEquals( 1, duh2.rollbackCommands.longValue() ); ureq.close(); // search - "B" should not be found. @@ -220,13 +220,13 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 { SolrQueryRequest ureq = req(); CommitUpdateCommand cmtCmd = new CommitUpdateCommand(ureq, false); cmtCmd.waitSearcher = true; - assertEquals( 2, duh2.addCommands.get() ); - assertEquals( 2, duh2.addCommandsCumulative.get() ); - assertEquals( 0, duh2.commitCommands.get() ); + assertEquals( 2, duh2.addCommands.longValue() ); + assertEquals( 2, duh2.addCommandsCumulative.longValue() ); + assertEquals( 0, duh2.commitCommands.longValue() ); updater.commit(cmtCmd); - assertEquals( 0, duh2.addCommands.get() ); - assertEquals( 2, duh2.addCommandsCumulative.get() ); - assertEquals( 1, duh2.commitCommands.get() ); + assertEquals( 0, duh2.addCommands.longValue() ); + assertEquals( 2, duh2.addCommandsCumulative.longValue() ); + assertEquals( 1, duh2.commitCommands.longValue() ); ureq.close(); // search - "A","B" should be found. @@ -253,14 +253,14 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 { // rollback "B" ureq = req(); RollbackUpdateCommand rbkCmd = new RollbackUpdateCommand(ureq); - assertEquals( 1, duh2.deleteByIdCommands.get() ); - assertEquals( 1, duh2.deleteByIdCommandsCumulative.get() ); - assertEquals( 0, duh2.rollbackCommands.get() ); + assertEquals( 1, duh2.deleteByIdCommands.longValue() ); + assertEquals( 1, duh2.deleteByIdCommandsCumulative.longValue() ); + assertEquals( 0, duh2.rollbackCommands.longValue() ); updater.rollback(rbkCmd); ureq.close(); - assertEquals( 0, duh2.deleteByIdCommands.get() ); - assertEquals( 0, duh2.deleteByIdCommandsCumulative.get() ); - assertEquals( 1, duh2.rollbackCommands.get() ); + assertEquals( 0, duh2.deleteByIdCommands.longValue() ); + assertEquals( 0, duh2.deleteByIdCommandsCumulative.longValue() ); + assertEquals( 1, duh2.rollbackCommands.longValue() ); // search - "B" should be found. assertQ("\"B\" should be found.", req diff --git a/solr/core/src/test/org/apache/solr/util/TestTestInjection.java b/solr/core/src/test/org/apache/solr/util/TestTestInjection.java index 418b4a48ee7..c4269ccbbf2 100644 --- a/solr/core/src/test/org/apache/solr/util/TestTestInjection.java +++ b/solr/core/src/test/org/apache/solr/util/TestTestInjection.java @@ -98,4 +98,8 @@ public class TestTestInjection extends LuceneTestCase { assertFalse(e.getMessage().toLowerCase(Locale.ENGLISH).contains("bad syntax")); } } + + public void testUsingConsistentRandomization() { + assertSame(random(), TestInjection.random()); + } } diff --git a/solr/licenses/curator-recipes-2.8.0.jar.sha1 b/solr/licenses/curator-recipes-2.8.0.jar.sha1 new file mode 100644 index 00000000000..82d894643cf --- /dev/null +++ b/solr/licenses/curator-recipes-2.8.0.jar.sha1 @@ -0,0 +1 @@ +c563e25fb37f85a6b029bc9746e75573640474fb diff --git a/solr/licenses/curator-recipes-LICENSE-ASL.txt b/solr/licenses/curator-recipes-LICENSE-ASL.txt new file mode 100644 index 00000000000..7a4a3ea2424 --- /dev/null +++ b/solr/licenses/curator-recipes-LICENSE-ASL.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/solr/licenses/curator-recipes-NOTICE.txt b/solr/licenses/curator-recipes-NOTICE.txt new file mode 100644 index 00000000000..f568d0fe32e --- /dev/null +++ b/solr/licenses/curator-recipes-NOTICE.txt @@ -0,0 +1,5 @@ +Apache Curator +Copyright 2013-2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/solr/solrj/ivy.xml b/solr/solrj/ivy.xml index ceefcc30809..f2296c55a7f 100644 --- a/solr/solrj/ivy.xml +++ b/solr/solrj/ivy.xml @@ -1,3 +1,4 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + diff --git a/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml b/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml new file mode 100644 index 00000000000..6b10869d8ce --- /dev/null +++ b/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml @@ -0,0 +1,51 @@ + + + + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + ${useCompoundFile:false} + + ${solr.data.dir:} + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + solr + + + + diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java index f2446f3128d..7c3a3a6e0c0 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java @@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.io.stream; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -26,6 +27,7 @@ import java.util.Map; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.io.ClassificationEvaluation; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.ComparatorOrder; @@ -71,6 +73,7 @@ public class StreamExpressionTest extends SolrCloudTestCase { public static void setupCluster() throws Exception { configureCluster(4) .addConfig("conf", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("streaming").resolve("conf")) + .addConfig("ml", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("ml").resolve("conf")) .configure(); CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 1).process(cluster.getSolrClient()); @@ -2773,6 +2776,8 @@ public class StreamExpressionTest extends SolrCloudTestCase { assert(tuple.getDouble("a_f") == 4.0); assertList(tuple.getStrings("s_multi"), "aaaa3", "bbbb3"); assertList(tuple.getLongs("i_multi"), Long.parseLong("4444"), Long.parseLong("7777")); + + CollectionAdminRequest.deleteCollection("destinationCollection").process(cluster.getSolrClient()); } @Test @@ -2863,6 +2868,7 @@ public class StreamExpressionTest extends SolrCloudTestCase { assertList(tuple.getStrings("s_multi"), "aaaa3", "bbbb3"); assertList(tuple.getLongs("i_multi"), Long.parseLong("4444"), Long.parseLong("7777")); + CollectionAdminRequest.deleteCollection("parallelDestinationCollection").process(cluster.getSolrClient()); } @Test @@ -3025,6 +3031,7 @@ public class StreamExpressionTest extends SolrCloudTestCase { assertList(tuple.getStrings("s_multi"), "aaaa3", "bbbb3"); assertList(tuple.getLongs("i_multi"), Long.parseLong("4444"), Long.parseLong("7777")); + CollectionAdminRequest.deleteCollection("parallelDestinationCollection1").process(cluster.getSolrClient()); } @Test @@ -3061,10 +3068,125 @@ public class StreamExpressionTest extends SolrCloudTestCase { tuples = getTuples(stream); assert(tuples.size() == 5); - assertOrder(tuples, 0,7,3,4,8); + assertOrder(tuples, 0, 7, 3, 4, 8); } + + @Test + public void testBasicTextLogitStream() throws Exception { + CollectionAdminRequest.createCollection("destinationCollection", "ml", 2, 1).process(cluster.getSolrClient()); + AbstractDistribZkTestBase.waitForRecoveriesToFinish("destinationCollection", cluster.getSolrClient().getZkStateReader(), + false, true, TIMEOUT); + + UpdateRequest updateRequest = new UpdateRequest(); + for (int i = 0; i < 5000; i+=2) { + updateRequest.add(id, String.valueOf(i), "tv_text", "a b c c d", "out_i", "1"); + updateRequest.add(id, String.valueOf(i+1), "tv_text", "a b e e f", "out_i", "0"); + } + updateRequest.commit(cluster.getSolrClient(), COLLECTION); + + StreamExpression expression; + TupleStream stream; + List tuples; + + StreamFactory factory = new StreamFactory() + .withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()) + .withCollectionZkHost("destinationCollection", cluster.getZkServer().getZkAddress()) + .withFunctionName("features", FeaturesSelectionStream.class) + .withFunctionName("train", TextLogitStream.class) + .withFunctionName("search", CloudSolrStream.class) + .withFunctionName("update", UpdateStream.class); + + expression = StreamExpressionParser.parse("features(collection1, q=\"*:*\", featureSet=\"first\", field=\"tv_text\", outcome=\"out_i\", numTerms=4)"); + stream = new FeaturesSelectionStream(expression, factory); + tuples = getTuples(stream); + + assert(tuples.size() == 4); + HashSet terms = new HashSet<>(); + for (Tuple tuple : tuples) { + terms.add((String) tuple.get("term_s")); + } + assertTrue(terms.contains("d")); + assertTrue(terms.contains("c")); + assertTrue(terms.contains("e")); + assertTrue(terms.contains("f")); + + String textLogitExpression = "train(" + + "collection1, " + + "features(collection1, q=\"*:*\", featureSet=\"first\", field=\"tv_text\", outcome=\"out_i\", numTerms=4),"+ + "q=\"*:*\", " + + "name=\"model\", " + + "field=\"tv_text\", " + + "outcome=\"out_i\", " + + "maxIterations=100)"; + stream = factory.constructStream(textLogitExpression); + tuples = getTuples(stream); + Tuple lastTuple = tuples.get(tuples.size() - 1); + List lastWeights = lastTuple.getDoubles("weights_ds"); + Double[] lastWeightsArray = lastWeights.toArray(new Double[lastWeights.size()]); + + // first feature is bias value + Double[] testRecord = {1.0, 1.17, 0.691, 0.0, 0.0}; + double d = sum(multiply(testRecord, lastWeightsArray)); + double prob = sigmoid(d); + assertEquals(prob, 1.0, 0.1); + + // first feature is bias value + Double[] testRecord2 = {1.0, 0.0, 0.0, 1.17, 0.691}; + d = sum(multiply(testRecord2, lastWeightsArray)); + prob = sigmoid(d); + assertEquals(prob, 0, 0.1); + + stream = factory.constructStream("update(destinationCollection, batchSize=5, "+textLogitExpression+")"); + getTuples(stream); + cluster.getSolrClient().commit("destinationCollection"); + + stream = factory.constructStream("search(destinationCollection, " + + "q=*:*, " + + "fl=\"iteration_i,* \", " + + "rows=100, " + + "sort=\"iteration_i desc\")"); + tuples = getTuples(stream); + assertEquals(100, tuples.size()); + Tuple lastModel = tuples.get(0); + ClassificationEvaluation evaluation = ClassificationEvaluation.create(lastModel.fields); + assertTrue(evaluation.getF1() >= 1.0); + assertEquals(Math.log( 5000.0 / (2500 + 1)), lastModel.getDoubles("idfs_ds").get(0), 0.0001); + // make sure the tuples is retrieved in correct order + Tuple firstTuple = tuples.get(99); + assertEquals(1L, (long) firstTuple.getLong("iteration_i")); + + CollectionAdminRequest.deleteCollection("destinationCollection").process(cluster.getSolrClient()); + } + + private double sigmoid(double in) { + + double d = 1.0 / (1+Math.exp(-in)); + return d; + } + + private double[] multiply(Double[] vec1, Double[] vec2) { + double[] working = new double[vec1.length]; + for(int i=0; i tuples; + + StreamFactory factory = new StreamFactory() + .withCollectionZkHost("collection1", cluster.getZkServer().getZkAddress()) + .withCollectionZkHost("destinationCollection", cluster.getZkServer().getZkAddress()) + .withFunctionName("featuresSelection", FeaturesSelectionStream.class) + .withFunctionName("search", CloudSolrStream.class) + .withFunctionName("update", UpdateStream.class); + + String featuresExpression = "featuresSelection(collection1, q=\"*:*\", featureSet=\"first\", field=\"whitetok\", outcome=\"out_i\", numTerms=4)"; + // basic + expression = StreamExpressionParser.parse(featuresExpression); + stream = new FeaturesSelectionStream(expression, factory); + tuples = getTuples(stream); + + assert(tuples.size() == 4); + + assertTrue(tuples.get(0).get("term_s").equals("c")); + assertTrue(tuples.get(1).get("term_s").equals("d")); + assertTrue(tuples.get(2).get("term_s").equals("e")); + assertTrue(tuples.get(3).get("term_s").equals("f")); + + // update + expression = StreamExpressionParser.parse("update(destinationCollection, batchSize=5, "+featuresExpression+")"); + stream = new UpdateStream(expression, factory); + getTuples(stream); + cluster.getSolrClient().commit("destinationCollection"); + + expression = StreamExpressionParser.parse("search(destinationCollection, q=featureSet_s:first, fl=\"index_i, term_s\", sort=\"index_i asc\")"); + stream = new CloudSolrStream(expression, factory); + tuples = getTuples(stream); + assertEquals(4, tuples.size()); + assertTrue(tuples.get(0).get("term_s").equals("c")); + assertTrue(tuples.get(1).get("term_s").equals("d")); + assertTrue(tuples.get(2).get("term_s").equals("e")); + assertTrue(tuples.get(3).get("term_s").equals("f")); + + CollectionAdminRequest.deleteCollection("destinationCollection").process(cluster.getSolrClient()); + } + @Test public void testComplementStream() throws Exception { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java index 63baa012adb..4ddf4ce8dce 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java @@ -62,6 +62,8 @@ public class StreamExpressionToExpessionTest extends LuceneTestCase { .withFunctionName("avg", MeanMetric.class) .withFunctionName("daemon", DaemonStream.class) .withFunctionName("topic", TopicStream.class) + .withFunctionName("tlogit", TextLogitStream.class) + .withFunctionName("featuresSelection", FeaturesSelectionStream.class) ; } @@ -138,7 +140,6 @@ public class StreamExpressionToExpessionTest extends LuceneTestCase { assertTrue(expressionString.contains("checkpointEvery=1000")); } - @Test public void testStatsStream() throws Exception { @@ -342,6 +343,40 @@ public class StreamExpressionToExpessionTest extends LuceneTestCase { assertTrue(firstExpressionString.contains("q=\"presentTitles:\\\"chief, executive officer\\\" AND age:[36 TO *]\"")); assertTrue(secondExpressionString.contains("q=\"presentTitles:\\\"chief, executive officer\\\" AND age:[36 TO *]\"")); } + + @Test + public void testFeaturesSelectionStream() throws Exception { + String expr = "featuresSelection(collection1, q=\"*:*\", featureSet=\"first\", field=\"tv_text\", outcome=\"out_i\", numTerms=4, positiveLabel=2)"; + FeaturesSelectionStream stream = new FeaturesSelectionStream(StreamExpressionParser.parse(expr), factory); + String expressionString = stream.toExpression(factory).toString(); + assertTrue(expressionString.contains("q=\"*:*\"")); + assertTrue(expressionString.contains("featureSet=first")); + assertTrue(expressionString.contains("field=tv_text")); + assertTrue(expressionString.contains("outcome=out_i")); + assertTrue(expressionString.contains("numTerms=4")); + assertTrue(expressionString.contains("positiveLabel=2")); + } + + @Test + public void testTextLogitStreamWithFeaturesSelection() throws Exception { + String expr = "tlogit(" + + "collection1, " + + "q=\"*:*\", " + + "name=\"model\", " + + "featuresSelection(collection1, q=\"*:*\", featureSet=\"first\", field=\"tv_text\", outcome=\"out_i\", numTerms=4), " + + "field=\"tv_text\", " + + "outcome=\"out_i\", " + + "maxIterations=100)"; + TextLogitStream logitStream = new TextLogitStream(StreamExpressionParser.parse(expr), factory); + String expressionString = logitStream.toExpression(factory).toString(); + assertTrue(expressionString.contains("q=\"*:*\"")); + assertTrue(expressionString.contains("name=model")); + assertFalse(expressionString.contains("terms=")); + assertTrue(expressionString.contains("featuresSelection(")); + assertTrue(expressionString.contains("field=tv_text")); + assertTrue(expressionString.contains("outcome=out_i")); + assertTrue(expressionString.contains("maxIterations=100")); + } @Test public void testCountMetric() throws Exception { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExplanationTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExplanationTest.java index f8765c9c2d7..91cab3dd557 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExplanationTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExplanationTest.java @@ -17,7 +17,6 @@ package org.apache.solr.client.solrj.io.stream; import junit.framework.Assert; - import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.io.ops.GroupOperation; import org.apache.solr.client.solrj.io.stream.expr.Explanation; diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestDelegationTokenRequest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestDelegationTokenRequest.java new file mode 100644 index 00000000000..47b8385cf06 --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestDelegationTokenRequest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.request; + +import org.apache.lucene.util.LuceneTestCase; + +import org.junit.Test; + +/** + * Test for DelegationTokenRequests + */ +public class TestDelegationTokenRequest extends LuceneTestCase { + + @Test + public void testGetRequest() throws Exception { + // without renewer + DelegationTokenRequest.Get get = new DelegationTokenRequest.Get(); + assertEquals("GETDELEGATIONTOKEN", get.getParams().get("op")); + assertNull(get.getParams().get("renewer")); + + + // with renewer + final String renewer = "test"; + get = new DelegationTokenRequest.Get(renewer); + assertEquals("GETDELEGATIONTOKEN", get.getParams().get("op")); + assertEquals(renewer, get.getParams().get("renewer")); + } + + @Test + public void testRenewRequest() throws Exception { + final String token = "testToken"; + DelegationTokenRequest.Renew renew = new DelegationTokenRequest.Renew(token); + assertEquals("RENEWDELEGATIONTOKEN", renew.getParams().get("op")); + assertEquals(token, renew.getParams().get("token")); + assertTrue(renew.getQueryParams().contains("op")); + assertTrue(renew.getQueryParams().contains("token")); + + // can handle null token + renew = new DelegationTokenRequest.Renew(null); + renew.getParams(); + } + + @Test + public void testCancelRequest() throws Exception { + final String token = "testToken"; + DelegationTokenRequest.Cancel cancel = new DelegationTokenRequest.Cancel(token); + assertEquals("CANCELDELEGATIONTOKEN", cancel.getParams().get("op")); + assertEquals(token, cancel.getParams().get("token")); + assertTrue(cancel.getQueryParams().contains("op")); + assertTrue(cancel.getQueryParams().contains("token")); + + // can handle null token + cancel = new DelegationTokenRequest.Cancel(null); + cancel.getParams(); + } +} diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/response/TestDelegationTokenResponse.java b/solr/solrj/src/test/org/apache/solr/client/solrj/response/TestDelegationTokenResponse.java new file mode 100644 index 00000000000..c376223b3e4 --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/response/TestDelegationTokenResponse.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.solrj.response; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.IOUtils; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.solr.client.solrj.ResponseParser; +import org.apache.solr.client.solrj.request.DelegationTokenRequest; +import org.apache.solr.common.SolrException; + +import org.junit.Test; + +import org.noggit.CharArr; +import org.noggit.JSONWriter; + +public class TestDelegationTokenResponse extends LuceneTestCase { + + private void delegationTokenResponse(DelegationTokenRequest request, + DelegationTokenResponse response, String responseBody) throws Exception { + ResponseParser parser = request.getResponseParser(); + response.setResponse(parser.processResponse( + IOUtils.toInputStream(responseBody, "UTF-8"), "UTF-8")); + } + + private String getNestedMapJson(String outerKey, String innerKey, Object innerValue) { + CharArr out = new CharArr(); + JSONWriter w = new JSONWriter(out, 2); + Map innerMap = new HashMap(); + innerMap.put(innerKey, innerValue); + Map> outerMap = new HashMap>(); + outerMap.put(outerKey, innerMap); + w.write(outerMap); + return out.toString(); + } + + private String getMapJson(String key, Object value) { + CharArr out = new CharArr(); + JSONWriter w = new JSONWriter(out, 2); + Map map = new HashMap(); + map.put(key, value); + w.write(map); + return out.toString(); + } + + @Test + public void testGetResponse() throws Exception { + DelegationTokenRequest.Get getRequest = new DelegationTokenRequest.Get(); + DelegationTokenResponse.Get getResponse = new DelegationTokenResponse.Get(); + + // not a map + try { + delegationTokenResponse(getRequest, getResponse, ""); + getResponse.getDelegationToken(); + fail("Expected SolrException"); + } catch (SolrException se) { + } + + // doesn't have Token outerMap + final String someToken = "someToken"; + delegationTokenResponse(getRequest, getResponse, getNestedMapJson("NotToken", "urlString", someToken)); + assertNull(getResponse.getDelegationToken()); + + // Token is not a map + try { + delegationTokenResponse(getRequest, getResponse, getMapJson("Token", someToken)); + getResponse.getDelegationToken(); + fail("Expected SolrException"); + } catch (SolrException se) { + } + + // doesn't have urlString + delegationTokenResponse(getRequest, getResponse, getNestedMapJson("Token", "notUrlString", someToken)); + assertNull(getResponse.getDelegationToken()); + + // has Token + urlString + delegationTokenResponse(getRequest, getResponse, getNestedMapJson("Token", "urlString", someToken)); + assertEquals(someToken, getResponse.getDelegationToken()); + } + + @Test + public void testRenewResponse() throws Exception { + DelegationTokenRequest.Renew renewRequest = new DelegationTokenRequest.Renew("token"); + DelegationTokenResponse.Renew renewResponse = new DelegationTokenResponse.Renew(); + + // not a map + try { + delegationTokenResponse(renewRequest, renewResponse, ""); + renewResponse.getExpirationTime(); + fail("Expected SolrException"); + } catch (SolrException se) { + } + + // doesn't have long + delegationTokenResponse(renewRequest, renewResponse, getMapJson("notLong", "123")); + assertNull(renewResponse.getExpirationTime()); + + // long isn't valid + try { + delegationTokenResponse(renewRequest, renewResponse, getMapJson("long", "aaa")); + renewResponse.getExpirationTime(); + fail("Expected SolrException"); + } catch (SolrException se) { + } + + // valid + Long expirationTime = new Long(Long.MAX_VALUE); + delegationTokenResponse(renewRequest, renewResponse, + getMapJson("long", expirationTime)); + assertEquals(expirationTime, renewResponse.getExpirationTime()); + } + + @Test + public void testCancelResponse() throws Exception { + // expect empty response + DelegationTokenRequest.Cancel cancelRequest = new DelegationTokenRequest.Cancel("token"); + DelegationTokenResponse.Cancel cancelResponse = new DelegationTokenResponse.Cancel(); + delegationTokenResponse(cancelRequest, cancelResponse, ""); + } +} diff --git a/solr/webapp/web/css/angular/cloud.css b/solr/webapp/web/css/angular/cloud.css index bbb66ae6a59..4017c221683 100644 --- a/solr/webapp/web/css/angular/cloud.css +++ b/solr/webapp/web/css/angular/cloud.css @@ -184,6 +184,20 @@ limitations under the License. background-image: url( ../../img/ico/cross-1.png ); } +#content #cloud #file-content #toggle.plus +{ + font-style: italic; + padding-left: 17px; + background-image: url( ../../img/ico/toggle-small-expand.png ); +} + +#content #cloud #file-content #toggle.minus +{ + font-style: italic; + padding-left: 17px; + background-image: url( ../../img/ico/toggle-small.png ); +} + #content #cloud #file-content #data { border-top: 1px solid #c0c0c0; diff --git a/solr/webapp/web/js/angular/app.js b/solr/webapp/web/js/angular/app.js index 8a1fe26c593..e7491fe63d4 100644 --- a/solr/webapp/web/js/angular/app.js +++ b/solr/webapp/web/js/angular/app.js @@ -148,7 +148,7 @@ solrAdminApp.config([ }) .filter('highlight', function($sce) { return function(input, lang) { - if (lang && input && lang!="text") return hljs.highlight(lang, input).value; + if (lang && input && lang!="txt") return hljs.highlight(lang, input).value; return input; } }) diff --git a/solr/webapp/web/js/angular/controllers/cloud.js b/solr/webapp/web/js/angular/controllers/cloud.js index c150c5a21ff..2d0dae0860b 100644 --- a/solr/webapp/web/js/angular/controllers/cloud.js +++ b/solr/webapp/web/js/angular/controllers/cloud.js @@ -55,6 +55,11 @@ var treeSubController = function($scope, Zookeeper) { var path = data.znode.path.split( '.' ); if(path.length >1) { $scope.lang = path.pop(); + } else { + var lastPathElement = data.znode.path.split( '/' ).pop(); + if (lastPathElement == "managed-schema") { + $scope.lang = "xml"; + } } $scope.showData = true; }); diff --git a/solr/webapp/web/js/angular/controllers/dataimport.js b/solr/webapp/web/js/angular/controllers/dataimport.js index 9ca723985da..a051ad2178b 100644 --- a/solr/webapp/web/js/angular/controllers/dataimport.js +++ b/solr/webapp/web/js/angular/controllers/dataimport.js @@ -61,6 +61,10 @@ solrAdminApp.controller('DataImportController', $scope.toggleDebug = function () { $scope.isDebugMode = !$scope.isDebugMode; + if ($scope.isDebugMode) { + // also enable Debug checkbox + $scope.form.showDebug = true; + } $scope.showConfiguration = true; } @@ -100,7 +104,13 @@ solrAdminApp.controller('DataImportController', $scope.submit = function () { var params = {}; for (var key in $scope.form) { - params[key] = $scope.form[key]; + if (key == "showDebug") { + if ($scope.form.showDebug) { + params["debug"] = true; + } + } else { + params[key] = $scope.form[key]; + } } if (params.custom.length) { var customParams = $scope.form.custom.split("&"); @@ -111,10 +121,10 @@ solrAdminApp.controller('DataImportController', } delete params.custom; - if (params.isDebugMode) { - params.dataConfig = $scope.rawConfig; + if ($scope.isDebugMode) { + params.dataConfig = $scope.config; } - delete params.showDebug; + params.core = $routeParams.core; DataImport.post(params, function (data) { diff --git a/solr/webapp/web/js/angular/controllers/query.js b/solr/webapp/web/js/angular/controllers/query.js index 8e89778acd1..1695647a3a0 100644 --- a/solr/webapp/web/js/angular/controllers/query.js +++ b/solr/webapp/web/js/angular/controllers/query.js @@ -61,9 +61,13 @@ solrAdminApp.controller('QueryController', if ($scope.rawParams) { var rawParams = $scope.rawParams.split(/[&\n]/); for (var i in rawParams) { - var param = rawParams[i]; - var parts = param.split("="); - set(parts[0], parts[1]); + var param = rawParams[i]; + var equalPos = param.indexOf("="); + if (equalPos > -1) { + set(param.substring(0, equalPos), param.substring(equalPos+1)); + } else { + set(param, ""); // Use empty value for params without "=" + } } } diff --git a/solr/webapp/web/partials/cloud.html b/solr/webapp/web/partials/cloud.html index ef39bb302c1..415e123e73d 100644 --- a/solr/webapp/web/partials/cloud.html +++ b/solr/webapp/web/partials/cloud.html @@ -23,7 +23,8 @@ limitations under the License.
-
+ Metadata +
  • diff --git a/solr/webapp/web/partials/dataimport.html b/solr/webapp/web/partials/dataimport.html index 5fde5975eaa..5cd6d2c3a0b 100644 --- a/solr/webapp/web/partials/dataimport.html +++ b/solr/webapp/web/partials/dataimport.html @@ -94,7 +94,7 @@ limitations under the License.
    - +
    @@ -115,11 +115,11 @@ limitations under the License.
- + No Request executed - -
+ +