Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

sequence numbers: merge master

This commit is contained in:
Mike McCandless 2016-05-26 15:30:12 -04:00
parent 8b0b0c9340 4d4030350b
commit 817c7f069a
93 changed files with 1491 additions and 898 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
dev-tools/scripts/smokeTestRelease.py
View File

@ -1192,7 +1192,7 @@ revision_re = re.compile(r'rev([a-f\d]+)')
def parse_config():
epilogue = textwrap.dedent('''
Example usage:
python3.2 -u dev-tools/scripts/smokeTestRelease.py http://people.apache.org/~whoever/staging_area/lucene-solr-4.3.0-RC1-rev1469340
python3 -u dev-tools/scripts/smokeTestRelease.py https://dist.apache.org/repos/dist/dev/lucene/lucene-solr-6.0.1-RC2-revc7510a0...
''')
description = 'Utility to test a release.'
parser = argparse.ArgumentParser(description=description, epilog=epilogue,
@ -1352,8 +1352,25 @@ def confirmAllReleasesAreTestedForBackCompat(smokeVersion, unpackPath):
else:
print(' success!')
def getScriptVersion():
topLevelDir = '../..' # Assumption: this script is in dev-tools/scripts/ of a checkout
m = re.compile(r'(.*)/').match(sys.argv[0]) # Get this script's directory
if m is not None and m.group(1) != '.':
origCwd = os.getcwd()
os.chdir(m.group(1))
os.chdir('../..')
topLevelDir = os.getcwd()
os.chdir(origCwd)
reBaseVersion = re.compile(r'version\.base\s*=\s*(\d+\.\d+)')
return reBaseVersion.search(open('%s/lucene/version.properties' % topLevelDir).read()).group(1)
def main():
c = parse_config()
scriptVersion = getScriptVersion()
if not c.version.startswith(scriptVersion + '.'):
raise RuntimeError('smokeTestRelease.py for %s.X is incompatible with a %s release.' % (scriptVersion, c.version))
print('NOTE: output encoding is %s' % sys.stdout.encoding)
smokeTest(c.java, c.url, c.revision, c.version, c.tmp_dir, c.is_signed, ' '.join(c.test_args))

12
lucene/CHANGES.txt
View File

@ -26,6 +26,10 @@ New Features
e.g. clear the Gregorian Change Date. Also, toString(cal) is now identical to
DateTimeFormatter.ISO_INSTANT. (David Smiley)
* LUCENE-7300: The misc module now has a directory wrapper that uses hard-links if
applicable and supported when copying files from another FSDirectory in
Directory#copyFrom. (Simon Willnauer)
API Changes
* LUCENE-7184: Refactor LatLonPoint encoding methods to new GeoEncodingUtils
@ -86,6 +90,8 @@ Optimizations
(which is used by TermsQuery, multi-term queries and several point queries).
(Adrien Grand, Jeff Wartes, David Smiley)
* LUCENE-7299: Speed up BytesRefHash.sort(). (Adrien Grand)
Bug Fixes
* LUCENE-7127: Fix corner case bugs in GeoPointDistanceQuery. (Robert Muir)
@ -136,6 +142,9 @@ Documentation
Other
* LUCENE-7295: TermAutomatonQuery.hashCode calculates Automaton.toDot().hash,
equivalence relationship replaced with object identity. (Dawid Weiss)
* LUCENE-7277: Make Query.hashCode and Query.equals abstract. (Paul Elschot,
Dawid Weiss)
@ -153,6 +162,9 @@ Other
* SOLR-9109/SOLR-9121: Allow specification of a custom Ivy settings file via system
property "ivysettings.xml". (Misha Dmitriev, Christine Poerschke, Uwe Schindler, Steve Rowe)
* LUCENE-7206: Improve the ToParentBlockJoinQuery's explain by including the explain
of the best matching child doc. (Ilya Kasnacheev, Jeff Evans via Martijn van Groningen)
Build
* LUCENE-7292: Use '-release' instead of '-source/-target' during

28
lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
View File

@ -145,18 +145,19 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
List<ClassificationResult<BytesRef>> assignedClasses = new ArrayList<>();
Terms classes = MultiFields.getTerms(leafReader, classFieldName);
TermsEnum classesEnum = classes.iterator();
BytesRef next;
String[] tokenizedText = tokenize(inputDocument);
int docsWithClassSize = countDocsWithClass();
while ((next = classesEnum.next()) != null) {
if (next.length > 0) {
Term term = new Term(this.classFieldName, next);
double clVal = calculateLogPrior(term, docsWithClassSize) + calculateLogLikelihood(tokenizedText, term, docsWithClassSize);
assignedClasses.add(new ClassificationResult<>(term.bytes(), clVal));
if (classes != null) {
TermsEnum classesEnum = classes.iterator();
BytesRef next;
String[] tokenizedText = tokenize(inputDocument);
int docsWithClassSize = countDocsWithClass();
while ((next = classesEnum.next()) != null) {
if (next.length > 0) {
Term term = new Term(this.classFieldName, next);
double clVal = calculateLogPrior(term, docsWithClassSize) + calculateLogLikelihood(tokenizedText, term, docsWithClassSize);
assignedClasses.add(new ClassificationResult<>(term.bytes(), clVal));
}
}
}
// normalization; the values transforms to a 0-1 range
return normClassificationResults(assignedClasses);
}
@ -168,8 +169,9 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
* @throws IOException if accessing to term vectors or search fails
*/
protected int countDocsWithClass() throws IOException {
int docCount = MultiFields.getTerms(this.leafReader, this.classFieldName).getDocCount();
if (docCount == -1) { // in case codec doesn't support getDocCount
Terms terms = MultiFields.getTerms(this.leafReader, this.classFieldName);
int docCount;
if (terms == null || terms.getDocCount() == -1) { // in case codec doesn't support getDocCount
TotalHitCountCollector classQueryCountCollector = new TotalHitCountCollector();
BooleanQuery.Builder q = new BooleanQuery.Builder();
q.add(new BooleanClause(new WildcardQuery(new Term(classFieldName, String.valueOf(WildcardQuery.WILDCARD_STRING))), BooleanClause.Occur.MUST));
@ -179,6 +181,8 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
indexSearcher.search(q.build(),
classQueryCountCollector);
docCount = classQueryCountCollector.getTotalHits();
} else {
docCount = terms.getDocCount();
}
return docCount;
}

22
lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
View File

@ -168,28 +168,6 @@ public class SimpleNaiveBayesDocumentClassifier extends SimpleNaiveBayesClassifi
}
}
/**
* Counts the number of documents in the index having at least a value for the 'class' field
*
* @return the no. of documents having a value for the 'class' field
* @throws java.io.IOException If accessing to term vectors or search fails
*/
protected int countDocsWithClass() throws IOException {
int docCount = MultiFields.getTerms(this.leafReader, this.classFieldName).getDocCount();
if (docCount == -1) { // in case codec doesn't support getDocCount
TotalHitCountCollector classQueryCountCollector = new TotalHitCountCollector();
BooleanQuery.Builder q = new BooleanQuery.Builder();
q.add(new BooleanClause(new WildcardQuery(new Term(classFieldName, String.valueOf(WildcardQuery.WILDCARD_STRING))), BooleanClause.Occur.MUST));
if (query != null) {
q.add(query, BooleanClause.Occur.MUST);
}
indexSearcher.search(q.build(),
classQueryCountCollector);
docCount = classQueryCountCollector.getTotalHits();
}
return docCount;
}
/**
* Returns a token array from the {@link org.apache.lucene.analysis.TokenStream} in input
*

49
lucene/classification/src/java/org/apache/lucene/classification/utils/ConfusionMatrixGenerator.java
View File

@ -175,7 +175,7 @@ public class ConfusionMatrixGenerator {
public double getPrecision(String klass) {
Map<String, Long> classifications = linearizedMatrix.get(klass);
double tp = 0;
double fp = 0;
double den = 0; // tp + fp
if (classifications != null) {
for (Map.Entry<String, Long> entry : classifications.entrySet()) {
if (klass.equals(entry.getKey())) {
@ -184,11 +184,11 @@ public class ConfusionMatrixGenerator {
}
for (Map<String, Long> values : linearizedMatrix.values()) {
if (values.containsKey(klass)) {
fp += values.get(klass);
den += values.get(klass);
}
}
}
return tp > 0 ? tp / (tp + fp) : 0;
return tp > 0 ? tp / den : 0;
}
/**
@ -246,7 +246,7 @@ public class ConfusionMatrixGenerator {
if (this.accuracy == -1) {
double tp = 0d;
double tn = 0d;
double fp = 0d;
double tfp = 0d; // tp + fp
double fn = 0d;
for (Map.Entry<String, Map<String, Long>> classification : linearizedMatrix.entrySet()) {
String klass = classification.getKey();
@ -259,63 +259,46 @@ public class ConfusionMatrixGenerator {
}
for (Map<String, Long> values : linearizedMatrix.values()) {
if (values.containsKey(klass)) {
fp += values.get(klass);
tfp += values.get(klass);
} else {
tn++;
}
}
}
this.accuracy = (tp + tn) / (fp + fn + tp + tn);
this.accuracy = (tp + tn) / (tfp + fn + tn);
}
return this.accuracy;
}
/**
* get the precision (see {@link #getPrecision(String)}) over all the classes.
* get the macro averaged precision (see {@link #getPrecision(String)}) over all the classes.
*
* @return the precision as computed from the whole confusion matrix
* @return the macro averaged precision as computed from the confusion matrix
*/
public double getPrecision() {
double tp = 0;
double fp = 0;
double p = 0;
for (Map.Entry<String, Map<String, Long>> classification : linearizedMatrix.entrySet()) {
String klass = classification.getKey();
for (Map.Entry<String, Long> entry : classification.getValue().entrySet()) {
if (klass.equals(entry.getKey())) {
tp += entry.getValue();
}
}
for (Map<String, Long> values : linearizedMatrix.values()) {
if (values.containsKey(klass)) {
fp += values.get(klass);
}
}
p += getPrecision(klass);
}
return tp > 0 ? tp / (tp + fp) : 0;
return p / linearizedMatrix.size();
}
/**
* get the recall (see {@link #getRecall(String)}) over all the classes
* get the macro averaged recall (see {@link #getRecall(String)}) over all the classes
*
* @return the recall as computed from the whole confusion matrix
* @return the recall as computed from the confusion matrix
*/
public double getRecall() {
double tp = 0;
double fn = 0;
double r = 0;
for (Map.Entry<String, Map<String, Long>> classification : linearizedMatrix.entrySet()) {
String klass = classification.getKey();
for (Map.Entry<String, Long> entry : classification.getValue().entrySet()) {
if (klass.equals(entry.getKey())) {
tp += entry.getValue();
} else {
fn += entry.getValue();
}
}
r += getRecall(klass);
}
return tp + fn > 0 ? tp / (tp + fn) : 0;
return r / linearizedMatrix.size();
}
@Override

1
lucene/classification/src/java/org/apache/lucene/classification/utils/DatasetSplitter.java
View File

@ -30,7 +30,6 @@ import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.ScoreDoc;

2
lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
View File

@ -661,7 +661,7 @@ public class MultiDocValues {
public final OrdinalMap mapping;
/** Creates a new MultiSortedDocValues over <code>values</code> */
MultiSortedDocValues(SortedDocValues values[], int docStarts[], OrdinalMap mapping) throws IOException {
public MultiSortedDocValues(SortedDocValues values[], int docStarts[], OrdinalMap mapping) throws IOException {
assert docStarts.length == values.length + 1;
this.values = values;
this.docStarts = docStarts;

268
lucene/core/src/java/org/apache/lucene/util/ArrayUtil.java
View File

@ -17,7 +17,6 @@
package org.apache.lucene.util;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
/**
@ -40,19 +39,6 @@ public final class ArrayUtil {
*/
/**
* Parses the string argument as if it was an int value and returns the
* result. Throws NumberFormatException if the string does not represent an
* int quantity.
*
* @param chars a string representation of an int quantity.
* @return int the value represented by the argument
* @throws NumberFormatException if the argument could not be parsed as an int quantity.
*/
public static int parseInt(char[] chars) throws NumberFormatException {
return parseInt(chars, 0, chars.length, 10);
}
/**
* Parses a char array into an int.
* @param chars the character array
@ -225,17 +211,6 @@ public final class ArrayUtil {
}
}
public static int getShrinkSize(int currentSize, int targetSize, int bytesPerElement) {
final int newSize = oversize(targetSize, bytesPerElement);
// Only reallocate if we are "substantially" smaller.
// This saves us from "running hot" (constantly making a
// bit bigger then a bit smaller, over and over):
if (newSize < currentSize / 2)
return newSize;
else
return currentSize;
}
public static <T> T[] grow(T[] array, int minSize) {
assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
@ -247,9 +222,7 @@ public final class ArrayUtil {
public static short[] grow(short[] array, int minSize) {
assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
short[] newArray = new short[oversize(minSize, Short.BYTES)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
return Arrays.copyOf(array, oversize(minSize, Short.BYTES));
} else
return array;
}
@ -261,9 +234,7 @@ public final class ArrayUtil {
public static float[] grow(float[] array, int minSize) {
assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
float[] newArray = new float[oversize(minSize, Float.BYTES)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
return Arrays.copyOf(array, oversize(minSize, Float.BYTES));
} else
return array;
}
@ -275,9 +246,7 @@ public final class ArrayUtil {
public static double[] grow(double[] array, int minSize) {
assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
double[] newArray = new double[oversize(minSize, Double.BYTES)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
return Arrays.copyOf(array, oversize(minSize, Double.BYTES));
} else
return array;
}
@ -286,23 +255,10 @@ public final class ArrayUtil {
return grow(array, 1 + array.length);
}
public static short[] shrink(short[] array, int targetSize) {
assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, Short.BYTES);
if (newSize != array.length) {
short[] newArray = new short[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else
return array;
}
public static int[] grow(int[] array, int minSize) {
assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
int[] newArray = new int[oversize(minSize, Integer.BYTES)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
return Arrays.copyOf(array, oversize(minSize, Integer.BYTES));
} else
return array;
}
@ -311,23 +267,10 @@ public final class ArrayUtil {
return grow(array, 1 + array.length);
}
public static int[] shrink(int[] array, int targetSize) {
assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, Integer.BYTES);
if (newSize != array.length) {
int[] newArray = new int[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else
return array;
}
public static long[] grow(long[] array, int minSize) {
assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
long[] newArray = new long[oversize(minSize, Long.BYTES)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
return Arrays.copyOf(array, oversize(minSize, Long.BYTES));
} else
return array;
}
@ -336,23 +279,10 @@ public final class ArrayUtil {
return grow(array, 1 + array.length);
}
public static long[] shrink(long[] array, int targetSize) {
assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, Long.BYTES);
if (newSize != array.length) {
long[] newArray = new long[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else
return array;
}
public static byte[] grow(byte[] array, int minSize) {
assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
byte[] newArray = new byte[oversize(minSize, 1)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
return Arrays.copyOf(array, oversize(minSize, Byte.BYTES));
} else
return array;
}
@ -361,48 +291,10 @@ public final class ArrayUtil {
return grow(array, 1 + array.length);
}
public static byte[] shrink(byte[] array, int targetSize) {
assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, 1);
if (newSize != array.length) {
byte[] newArray = new byte[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else
return array;
}
public static boolean[] grow(boolean[] array, int minSize) {
assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
boolean[] newArray = new boolean[oversize(minSize, 1)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
} else
return array;
}
public static boolean[] grow(boolean[] array) {
return grow(array, 1 + array.length);
}
public static boolean[] shrink(boolean[] array, int targetSize) {
assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, 1);
if (newSize != array.length) {
boolean[] newArray = new boolean[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else
return array;
}
public static char[] grow(char[] array, int minSize) {
assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
char[] newArray = new char[oversize(minSize, Character.BYTES)];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
return Arrays.copyOf(array, oversize(minSize, Character.BYTES));
} else
return array;
}
@ -411,71 +303,6 @@ public final class ArrayUtil {
return grow(array, 1 + array.length);
}
public static char[] shrink(char[] array, int targetSize) {
assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, Character.BYTES);
if (newSize != array.length) {
char[] newArray = new char[newSize];
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else
return array;
}
public static int[][] grow(int[][] array, int minSize) {
assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
int[][] newArray = new int[oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF)][];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
} else {
return array;
}
}
public static int[][] grow(int[][] array) {
return grow(array, 1 + array.length);
}
public static int[][] shrink(int[][] array, int targetSize) {
assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
if (newSize != array.length) {
int[][] newArray = new int[newSize][];
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else {
return array;
}
}
public static float[][] grow(float[][] array, int minSize) {
assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
float[][] newArray = new float[oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF)][];
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
} else {
return array;
}
}
public static float[][] grow(float[][] array) {
return grow(array, 1 + array.length);
}
public static float[][] shrink(float[][] array, int targetSize) {
assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
if (newSize != array.length) {
float[][] newArray = new float[newSize][];
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else {
return array;
}
}
/**
* Returns hash of chars in range start (inclusive) to
* end (inclusive)
@ -486,44 +313,6 @@ public final class ArrayUtil {
code = code * 31 + array[i];
return code;
}
/**
* Returns hash of bytes in range start (inclusive) to
* end (inclusive)
*/
public static int hashCode(byte[] array, int start, int end) {
int code = 0;
for (int i = end - 1; i >= start; i--)
code = code * 31 + array[i];
return code;
}
// Since Arrays.equals doesn't implement offsets for equals
/**
* See if two array slices are the same.
*
* @param left The left array to compare
* @param offsetLeft The offset into the array. Must be positive
* @param right The right array to compare
* @param offsetRight the offset into the right array. Must be positive
* @param length The length of the section of the array to compare
* @return true if the two arrays, starting at their respective offsets, are equal
*
* @see java.util.Arrays#equals(char[], char[])
*/
public static boolean equals(char[] left, int offsetLeft, char[] right, int offsetRight, int length) {
if ((offsetLeft + length <= left.length) && (offsetRight + length <= right.length)) {
for (int i = 0; i < length; i++) {
if (left[offsetLeft + i] != right[offsetRight + i]) {
return false;
}
}
return true;
}
return false;
}
// Since Arrays.equals doesn't implement offsets for equals
/**
@ -551,35 +340,6 @@ public final class ArrayUtil {
return false;
}
/* DISABLE THIS FOR NOW: This has performance problems until Java creates intrinsics for Class#getComponentType() and Array.newInstance()
public static <T> T[] grow(T[] array, int minSize) {
assert minSize >= 0: "size must be positive (got " + minSize + "): likely integer overflow?";
if (array.length < minSize) {
@SuppressWarnings("unchecked") final T[] newArray =
(T[]) Array.newInstance(array.getClass().getComponentType(), oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
System.arraycopy(array, 0, newArray, 0, array.length);
return newArray;
} else
return array;
}
public static <T> T[] grow(T[] array) {
return grow(array, 1 + array.length);
}
public static <T> T[] shrink(T[] array, int targetSize) {
assert targetSize >= 0: "size must be positive (got " + targetSize + "): likely integer overflow?";
final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
if (newSize != array.length) {
@SuppressWarnings("unchecked") final T[] newArray =
(T[]) Array.newInstance(array.getClass().getComponentType(), newSize);
System.arraycopy(array, 0, newArray, 0, newSize);
return newArray;
} else
return array;
}
*/
// Since Arrays.equals doesn't implement offsets for equals
/**
* See if two array slices are the same.
@ -606,20 +366,6 @@ public final class ArrayUtil {
return false;
}
public static int[] toIntArray(Collection<Integer> ints) {
final int[] result = new int[ints.size()];
int upto = 0;
for(int v : ints) {
result[upto++] = v;
}
// paranoia:
assert upto == result.length;
return result;
}
/** Swap values stored in slots <code>i</code> and <code>j</code> */
public static <T> void swap(T[] arr, int i, int j) {
final T tmp = arr[i];

37
lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java
View File

@ -158,40 +158,23 @@ public final class BytesRefHash {
*/
public int[] sort() {
final int[] compact = compact();
new IntroSorter() {
new StringMSBRadixSorter() {
BytesRef scratch = new BytesRef();
@Override
protected void swap(int i, int j) {
final int o = compact[i];
int tmp = compact[i];
compact[i] = compact[j];
compact[j] = o;
}
@Override
protected int compare(int i, int j) {
final int id1 = compact[i], id2 = compact[j];
assert bytesStart.length > id1 && bytesStart.length > id2;
pool.setBytesRef(scratch1, bytesStart[id1]);
pool.setBytesRef(scratch2, bytesStart[id2]);
return scratch1.compareTo(scratch2);
compact[j] = tmp;
}
@Override
protected void setPivot(int i) {
final int id = compact[i];
assert bytesStart.length > id;
pool.setBytesRef(pivot, bytesStart[id]);
protected BytesRef get(int i) {
pool.setBytesRef(scratch, bytesStart[compact[i]]);
return scratch;
}
@Override
protected int comparePivot(int j) {
final int id = compact[j];
assert bytesStart.length > id;
pool.setBytesRef(scratch2, bytesStart[id]);
return pivot.compareTo(scratch2);
}
private final BytesRef pivot = new BytesRef(),
scratch1 = new BytesRef(), scratch2 = new BytesRef();
}.sort(0, count);
return compact;
}

2
lucene/core/src/java/org/apache/lucene/util/InPlaceMergeSorter.java
View File

@ -33,7 +33,7 @@ public abstract class InPlaceMergeSorter extends Sorter {
}
void mergeSort(int from, int to) {
if (to - from < THRESHOLD) {
if (to - from < INSERTION_SORT_THRESHOLD) {
insertionSort(from, to);
} else {
final int mid = (from + to) >>> 1;

2
lucene/core/src/java/org/apache/lucene/util/IntroSorter.java
View File

@ -38,7 +38,7 @@ public abstract class IntroSorter extends Sorter {
}
void quicksort(int from, int to, int maxDepth) {
if (to - from < THRESHOLD) {
if (to - from < INSERTION_SORT_THRESHOLD) {
insertionSort(from, to);
return;
} else if (--maxDepth < 0) {

2
lucene/core/src/java/org/apache/lucene/util/Sorter.java
View File

@ -23,7 +23,7 @@ import java.util.Comparator;
* @lucene.internal */
public abstract class Sorter {
static final int THRESHOLD = 20;
static final int INSERTION_SORT_THRESHOLD = 20;
/** Sole constructor, used for inheritance. */
protected Sorter() {}

190
lucene/core/src/java/org/apache/lucene/util/StringMSBRadixSorter.java Normal file
View File

@ -0,0 +1,190 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util;
import java.util.Arrays;
/** Radix sorter for variable-length strings. This class sorts based on the most
* significant byte first and falls back to {@link IntroSorter} when the size
* of the buckets to sort becomes small. It is <b>NOT</b> stable.
* Worst-case memory usage is about {@code 2.3 KB}. */
abstract class StringMSBRadixSorter extends Sorter {
// after that many levels of recursion we fall back to introsort anyway
// this is used as a protection against the fact that radix sort performs
// worse when there are long common prefixes (probably because of cache
// locality)
private static final int LEVEL_THRESHOLD = 8;
// size of histograms: 256 + 1 to indicate that the string is finished
private static final int HISTOGRAM_SIZE = 257;
// buckets below this size will be sorted with introsort
private static final int LENGTH_THRESHOLD = 100;
// we store one histogram per recursion level
private final int[][] histograms = new int[LEVEL_THRESHOLD][];
private final int[] endOffsets = new int[HISTOGRAM_SIZE];
/** Get a {@link BytesRef} for the given index. */
protected abstract BytesRef get(int i);
/** Store bytes for the given index into {@code dest}, without the first k bytes. */
private void get(int i, int k, BytesRef dest) {
BytesRef ref = get(i);
assert ref.length >= k;
dest.bytes = ref.bytes;
dest.offset = ref.offset + k;
dest.length = ref.length - k;
}
@Override
protected final int compare(int i, int j) {
throw new UnsupportedOperationException("unused: not a comparison-based sort");
}
@Override
public void sort(int from, int to) {
checkRange(from, to);
sort(from, to, 0);
}
private void sort(int from, int to, int k) {
if (to - from <= LENGTH_THRESHOLD || k >= LEVEL_THRESHOLD) {
introSort(from, to, k);
} else {
radixSort(from, to, k);
}
}
private void introSort(int from, int to, int k) {
new IntroSorter() {
@Override
protected void swap(int i, int j) {
StringMSBRadixSorter.this.swap(i, j);
}
@Override
protected int compare(int i, int j) {
get(i, k, scratch1);
get(j, k, scratch2);
return scratch1.compareTo(scratch2);
}
@Override
protected void setPivot(int i) {
get(i, k, pivot);
}
@Override
protected int comparePivot(int j) {
get(j, k, scratch2);
return pivot.compareTo(scratch2);
}
private final BytesRef pivot = new BytesRef(),
scratch1 = new BytesRef(), scratch2 = new BytesRef();
}.sort(from, to);
}
private void radixSort(int from, int to, int k) {
int[] histogram = histograms[k];
if (histogram == null) {
histogram = histograms[k] = new int[HISTOGRAM_SIZE];
} else {
Arrays.fill(histogram, 0);
}
buildHistogram(from, to, k, histogram);
// short-circuit: if all keys have the same byte at offset k, then recurse directly
for (int i = 0; i < HISTOGRAM_SIZE; ++i) {
if (histogram[i] == to - from) {
// everything is in the same bucket, recurse
if (i > 0) {
sort(from, to, k + 1);
}
return;
} else if (histogram[i] != 0) {
break;
}
}
int[] startOffsets = histogram;
int[] endOffsets = this.endOffsets;
sumHistogram(histogram, endOffsets);
reorder(from, to, startOffsets, endOffsets, k);
endOffsets = startOffsets;
// recurse on all but the first bucket since all keys are equals in this
// bucket (we already compared all bytes)
for (int prev = endOffsets[0], i = 1; i < HISTOGRAM_SIZE; ++i) {
int h = endOffsets[i];
final int bucketLen = h - prev;
if (bucketLen > 1) {
sort(from + prev, from + h, k + 1);
}
prev = h;
}
}
/** Return a number for the k-th character between 0 and {@link #HISTOGRAM_SIZE}. */
private int getBucket(int id, int k) {
BytesRef ref = get(id);
if (ref.length <= k) {
return 0;
}
final int b = ref.bytes[ref.offset + k] & 0xff;
return b + 1;
}
/** Build a histogram of the number of values per {@link #getBucket(int, int) bucket}. */
private int[] buildHistogram(int from, int to, int k, int[] histogram) {
for (int i = from; i < to; ++i) {
histogram[getBucket(i, k)]++;
}
return histogram;
}
/** Accumulate values of the histogram so that it does not store counts but
* start offsets. {@code endOffsets} will store the end offsets. */
private static void sumHistogram(int[] histogram, int[] endOffsets) {
int accum = 0;
for (int i = 0; i < HISTOGRAM_SIZE; ++i) {
final int count = histogram[i];
histogram[i] = accum;
accum += count;
endOffsets[i] = accum;
}
}
/**
* Reorder based on start/end offsets for each bucket. When this method
* returns, startOffsets and endOffsets are equal.
* @param startOffsets start offsets per bucket
* @param endOffsets end offsets per bucket
*/
private void reorder(int from, int to, int[] startOffsets, int[] endOffsets, int k) {
// reorder in place, like the dutch flag problem
for (int i = 0; i < HISTOGRAM_SIZE; ++i) {
final int limit = endOffsets[i];
for (int h1 = startOffsets[i]; h1 < limit; h1 = startOffsets[i]) {
final int b = getBucket(from + h1, k);
final int h2 = startOffsets[b]++;
swap(from + h1, from + h2);
}
}
}
}

21
lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
View File

@ -18,6 +18,7 @@ package org.apache.lucene.util.bkd;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.lucene.util.ArrayUtil;
@ -105,20 +106,6 @@ final class HeapPointWriter implements PointWriter {
System.arraycopy(bytes, 0, blocks.get(block), blockIndex * packedBytesLength, packedBytesLength);
}
private int[] growExact(int[] arr, int size) {
assert size > arr.length;
int[] newArr = new int[size];
System.arraycopy(arr, 0, newArr, 0, arr.length);
return newArr;
}
private long[] growExact(long[] arr, int size) {
assert size > arr.length;
long[] newArr = new long[size];
System.arraycopy(arr, 0, newArr, 0, arr.length);
return newArr;
}
@Override
public void append(byte[] packedValue, long ord, int docID) {
assert closed == false;
@ -126,12 +113,12 @@ final class HeapPointWriter implements PointWriter {
if (docIDs.length == nextWrite) {
int nextSize = Math.min(maxSize, ArrayUtil.oversize(nextWrite+1, Integer.BYTES));
assert nextSize > nextWrite: "nextSize=" + nextSize + " vs nextWrite=" + nextWrite;
docIDs = growExact(docIDs, nextSize);
docIDs = Arrays.copyOf(docIDs, nextSize);
if (singleValuePerDoc == false) {
if (ordsLong != null) {
ordsLong = growExact(ordsLong, nextSize);
ordsLong = Arrays.copyOf(ordsLong, nextSize);
} else {
ords = growExact(ords, nextSize);
ords = Arrays.copyOf(ords, nextSize);
}
}
}

9
lucene/core/src/java/org/apache/lucene/util/fst/Builder.java
View File

@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.fst.FST.INPUT_TYPE; // javadoc
import org.apache.lucene.util.packed.PackedInts;
@ -403,9 +402,7 @@ public class Builder<T> {
final int prefixLenPlus1 = pos1+1;
if (frontier.length < input.length+1) {
@SuppressWarnings({"rawtypes","unchecked"}) final UnCompiledNode<T>[] next =
new UnCompiledNode[ArrayUtil.oversize(input.length+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(frontier, 0, next, 0, frontier.length);
final UnCompiledNode<T>[] next = ArrayUtil.grow(frontier, input.length+1);
for(int idx=frontier.length;idx<next.length;idx++) {
next[idx] = new UnCompiledNode<>(this, idx);
}
@ -606,9 +603,7 @@ public class Builder<T> {
assert label >= 0;
assert numArcs == 0 || label > arcs[numArcs-1].label: "arc[-1].label=" + arcs[numArcs-1].label + " new label=" + label + " numArcs=" + numArcs;
if (numArcs == arcs.length) {
@SuppressWarnings({"rawtypes","unchecked"}) final Arc<T>[] newArcs =
new Arc[ArrayUtil.oversize(numArcs+1, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(arcs, 0, newArcs, 0, arcs.length);
final Arc<T>[] newArcs = ArrayUtil.grow(arcs, numArcs+1);
for(int arcIdx=numArcs;arcIdx<newArcs.length;arcIdx++) {
newArcs[arcIdx] = new Arc<>();
}

9
lucene/core/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Random;
@ -233,12 +234,14 @@ public class TestPerSegmentDeletes extends LuceneTestCase {
}
public static int[] toArray(PostingsEnum postingsEnum) throws IOException {
List<Integer> docs = new ArrayList<>();
int[] docs = new int[0];
int numDocs = 0;
while (postingsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
int docID = postingsEnum.docID();
docs.add(docID);
docs = ArrayUtil.grow(docs, numDocs + 1);
docs[numDocs + 1] = docID;
}
return ArrayUtil.toIntArray(docs);
return Arrays.copyOf(docs, numDocs);
}
public class RangeMergePolicy extends MergePolicy {

28
lucene/core/src/test/org/apache/lucene/util/TestArrayUtil.java
View File

@ -17,6 +17,7 @@
package org.apache.lucene.util;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.Random;
@ -70,30 +71,37 @@ public class TestArrayUtil extends LuceneTestCase {
}
}
private static int parseInt(String s) {
int start = random().nextInt(5);
char[] chars = new char[s.length() + start + random().nextInt(4)];
s.getChars(0, s.length(), chars, start);
return ArrayUtil.parseInt(chars, start, s.length());
}
public void testParseInt() throws Exception {
expectThrows(NumberFormatException.class, () -> {
ArrayUtil.parseInt("".toCharArray());
parseInt("");
});
expectThrows(NumberFormatException.class, () -> {
ArrayUtil.parseInt("foo".toCharArray());
parseInt("foo");
});
expectThrows(NumberFormatException.class, () -> {
ArrayUtil.parseInt(String.valueOf(Long.MAX_VALUE).toCharArray());
parseInt(String.valueOf(Long.MAX_VALUE));
});
expectThrows(NumberFormatException.class, () -> {
ArrayUtil.parseInt("0.34".toCharArray());
parseInt("0.34");
});
int test = ArrayUtil.parseInt("1".toCharArray());
int test = parseInt("1");
assertTrue(test + " does not equal: " + 1, test == 1);
test = ArrayUtil.parseInt("-10000".toCharArray());
test = parseInt("-10000");
assertTrue(test + " does not equal: " + -10000, test == -10000);
test = ArrayUtil.parseInt("1923".toCharArray());
test = parseInt("1923");
assertTrue(test + " does not equal: " + 1923, test == 1923);
test = ArrayUtil.parseInt("-1".toCharArray());
test = parseInt("-1");
assertTrue(test + " does not equal: " + -1, test == -1);
test = ArrayUtil.parseInt("foo 1923 bar".toCharArray(), 4, 4);
assertTrue(test + " does not equal: " + 1923, test == 1923);
@ -102,8 +110,8 @@ public class TestArrayUtil extends LuceneTestCase {
public void testSliceEquals() {
String left = "this is equal";
String right = left;
char[] leftChars = left.toCharArray();
char[] rightChars = right.toCharArray();
byte[] leftChars = left.getBytes(StandardCharsets.UTF_8);
byte[] rightChars = right.getBytes(StandardCharsets.UTF_8);
assertTrue(left + " does not equal: " + right, ArrayUtil.equals(leftChars, 0, rightChars, 0, left.length()));
assertFalse(left + " does not equal: " + right, ArrayUtil.equals(leftChars, 1, rightChars, 0, left.length()));

97
lucene/core/src/test/org/apache/lucene/util/TestStringMSBRadixSorter.java Normal file
View File

@ -0,0 +1,97 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util;
import java.util.Arrays;
public class TestStringMSBRadixSorter extends LuceneTestCase {
private void test(BytesRef[] refs, int len) {
BytesRef[] expected = Arrays.copyOf(refs, len);
Arrays.sort(expected);
new StringMSBRadixSorter() {
@Override
protected BytesRef get(int i) {
return refs[i];
}
@Override
protected void swap(int i, int j) {
BytesRef tmp = refs[i];
refs[i] = refs[j];
refs[j] = tmp;
}
}.sort(0, len);
BytesRef[] actual = Arrays.copyOf(refs, len);
assertArrayEquals(expected, actual);
}
public void testEmpty() {
test(new BytesRef[random().nextInt(5)], 0);
}
public void testOneValue() {
BytesRef bytes = new BytesRef(TestUtil.randomSimpleString(random()));
test(new BytesRef[] { bytes }, 1);
}
public void testTwoValues() {
BytesRef bytes1 = new BytesRef(TestUtil.randomSimpleString(random()));
BytesRef bytes2 = new BytesRef(TestUtil.randomSimpleString(random()));
test(new BytesRef[] { bytes1, bytes2 }, 2);
}
private void testRandom(int commonPrefixLen, int maxLen) {
byte[] commonPrefix = new byte[commonPrefixLen];
random().nextBytes(commonPrefix);
final int len = random().nextInt(100000);
BytesRef[] bytes = new BytesRef[len + random().nextInt(50)];
for (int i = 0; i < len; ++i) {
byte[] b = new byte[commonPrefixLen + random().nextInt(maxLen)];
random().nextBytes(b);
System.arraycopy(commonPrefix, 0, b, 0, commonPrefixLen);
bytes[i] = new BytesRef(b);
}
test(bytes, len);
}
public void testRandom() {
for (int iter = 0; iter < 10; ++iter) {
testRandom(0, 10);
}
}
public void testRandomWithLotsOfDuplicates() {
for (int iter = 0; iter < 10; ++iter) {
testRandom(0, 2);
}
}
public void testRandomWithSharedPrefix() {
for (int iter = 0; iter < 10; ++iter) {
testRandom(TestUtil.nextInt(random(), 1, 30), 10);
}
}
public void testRandomWithSharedPrefixAndLotsOfDuplicates() {
for (int iter = 0; iter < 10; ++iter) {
testRandom(TestUtil.nextInt(random(), 1, 30), 2);
}
}
}

4
lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java
View File

@ -280,13 +280,13 @@ public class ToParentBlockJoinCollector implements Collector {
// While rare, this could happen if join query had
// null scorer on first segment(s) but then became
// non-null on later segments
og.docs = ArrayUtil.grow(og.docs);
og.docs = ArrayUtil.grow(og.docs, numSubScorers);
}
if (og.counts.length < numSubScorers) {
og.counts = ArrayUtil.grow(og.counts);
}
if (trackScores && og.scores.length < numSubScorers) {
og.scores = ArrayUtil.grow(og.scores);
og.scores = ArrayUtil.grow(og.scores, numSubScorers);
}
//System.out.println("\ncopyGroups parentDoc=" + og.doc);

26
lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java
View File

@ -17,8 +17,10 @@
package org.apache.lucene.search.join;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Set;
@ -184,7 +186,7 @@ public class ToParentBlockJoinQuery extends Query {
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
BlockJoinScorer scorer = (BlockJoinScorer) scorer(context);
if (scorer != null && scorer.iterator().advance(doc) == doc) {
return scorer.explain(context.docBase);
return scorer.explain(context, childWeight);
}
return Explanation.noMatch("Not a match");
}
@ -436,10 +438,24 @@ public class ToParentBlockJoinQuery extends Query {
return parentFreq;
}
public Explanation explain(int docBase) throws IOException {
int start = docBase + prevParentDoc + 1; // +1 b/c prevParentDoc is previous parent doc
int end = docBase + parentDoc - 1; // -1 b/c parentDoc is parent doc
return Explanation.match(score(), String.format(Locale.ROOT, "Score based on child doc range from %d to %d", start, end)
public Explanation explain(LeafReaderContext context, Weight childWeight) throws IOException {
int start = context.docBase + prevParentDoc + 1; // +1 b/c prevParentDoc is previous parent doc
int end = context.docBase + parentDoc - 1; // -1 b/c parentDoc is parent doc
Explanation bestChild = null;
int matches = 0;
for (int childDoc = start; childDoc <= end; childDoc++) {
Explanation child = childWeight.explain(context, childDoc - context.docBase);
if (child.isMatch()) {
matches++;
if (bestChild == null || child.getValue() > bestChild.getValue()) {
bestChild = child;
}
}
}
return Explanation.match(score(), String.format(Locale.ROOT,
"Score based on %d child docs in range from %d to %d, best match:", matches, start, end), bestChild
);
}

16
lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
View File

@ -24,6 +24,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@ -50,6 +52,7 @@ import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.CheckHits;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.FieldDoc;
@ -231,6 +234,8 @@ public class TestBlockJoin extends LuceneTestCase {
ToParentBlockJoinCollector c = new ToParentBlockJoinCollector(Sort.RELEVANCE, 1, true, true);
CheckHits.checkHitCollector(random(), fullQuery.build(), "country", s, new int[] {2});
s.search(fullQuery.build(), c);
TopGroups<Integer> results = c.getTopGroups(childJoinQuery, null, 0, 10, 0, true);
@ -869,7 +874,16 @@ public class TestBlockJoin extends LuceneTestCase {
//System.out.println(" hit docID=" + hit.doc + " childId=" + childId + " parentId=" + document.get("parentID"));
assertTrue(explanation.isMatch());
assertEquals(hit.score, explanation.getValue(), 0.0f);
assertEquals(String.format(Locale.ROOT, "Score based on child doc range from %d to %d", hit.doc - 1 - childId, hit.doc - 1), explanation.getDescription());
Matcher m = Pattern.compile("Score based on ([0-9]+) child docs in range from ([0-9]+) to ([0-9]+), best match:").matcher(explanation.getDescription());
assertTrue("Block Join description not matches", m.matches());
assertTrue("Matched children not positive", Integer.parseInt(m.group(1)) > 0);
assertEquals("Wrong child range start", hit.doc - 1 - childId, Integer.parseInt(m.group(2)));
assertEquals("Wrong child range end", hit.doc - 1, Integer.parseInt(m.group(3)));
Explanation childWeightExplanation = explanation.getDetails()[0];
if ("sum of:".equals(childWeightExplanation.getDescription())) {
childWeightExplanation = childWeightExplanation.getDetails()[0];
}
assertTrue("Wrong child weight description", childWeightExplanation.getDescription().startsWith("weight(child"));
}
}

4
lucene/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java
View File

@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.HardlinkCopyDirectoryWrapper;
import org.apache.lucene.util.SuppressForbidden;
import java.io.IOException;
@ -45,7 +46,8 @@ public class IndexMergeTool {
Directory[] indexes = new Directory[args.length - 1];
for (int i = 1; i < args.length; i++) {
indexes[i - 1] = FSDirectory.open(Paths.get(args[i]));
// try to use hardlinks if possible
indexes[i - 1] = new HardlinkCopyDirectoryWrapper(FSDirectory.open(Paths.get(args[i])));
}
System.out.println("Merging...");

95
lucene/misc/src/java/org/apache/lucene/store/HardlinkCopyDirectoryWrapper.java Normal file
View File

@ -0,0 +1,95 @@
package org.apache.lucene.store;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.security.AccessController;
import java.security.PrivilegedAction;
/**
* This directory wrapper overrides {@link Directory#copyFrom(Directory, String, String, IOContext)} in order
* to optionally use a hard-link instead of a full byte by byte file copy if applicable. Hard-links are only used if the
* underlying filesystem supports it and if the {@link java.nio.file.LinkPermission} "hard" is granted.
*
* <p><b>NOTE:</b> Using hard-links changes the copy semantics of
* {@link Directory#copyFrom(Directory, String, String, IOContext)}. When hard-links are used changes to the source file
* will be reflected in the target file and vice-versa. Within Lucene, files are write once and should not be modified
* after they have been written. This directory should not be used in situations where files change after they have
* been written.
* </p>
*/
public final class HardlinkCopyDirectoryWrapper extends FilterDirectory {
/**
* Creates a new HardlinkCopyDirectoryWrapper delegating to the given directory
*/
public HardlinkCopyDirectoryWrapper(Directory in) {
super(in);
}
@Override
public void copyFrom(Directory from, String srcFile, String destFile, IOContext context) throws IOException {
final Directory fromUnwrapped = FilterDirectory.unwrap(from);
final Directory toUnwrapped = FilterDirectory.unwrap(this);
// try to unwrap to FSDirectory - we might be able to just create hard-links of these files and save copying
// the entire file.
Exception suppressedException = null;
boolean tryCopy = true;
if (fromUnwrapped instanceof FSDirectory
&& toUnwrapped instanceof FSDirectory) {
final Path fromPath = ((FSDirectory) fromUnwrapped).getDirectory();
final Path toPath = ((FSDirectory) toUnwrapped).getDirectory();
if (Files.isReadable(fromPath.resolve(srcFile)) && Files.isWritable(toPath)) {
// only try hardlinks if we have permission to access the files
// if not super.copyFrom() will give us the right exceptions
suppressedException = AccessController.doPrivileged((PrivilegedAction<Exception>) () -> {
try {
Files.createLink(toPath.resolve(destFile), fromPath.resolve(srcFile));
} catch (FileNotFoundException | NoSuchFileException | FileAlreadyExistsException ex) {
return ex; // in these cases we bubble up since it's a true error condition.
} catch (IOException
| UnsupportedOperationException // if the FS doesn't support hard-links
| SecurityException ex // we don't have permission to use hard-links just fall back to byte copy
) {
// hard-links are not supported or the files are on different filesystems
// we could go deeper and check if their filesstores are the same and opt
// out earlier but for now we just fall back to normal file-copy
return ex;
}
return null;
});
tryCopy = suppressedException != null;
}
}
if (tryCopy) {
try {
super.copyFrom(from, srcFile, destFile, context);
} catch (Exception ex) {
if (suppressedException != null) {
ex.addSuppressed(suppressedException);
}
throw ex;
}
}
}
}

81
lucene/misc/src/test/org/apache/lucene/store/TestHardLinkCopyDirectoryWrapper.java Normal file
View File

@ -0,0 +1,81 @@
package org.apache.lucene.store;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.util.IOUtils;
public class TestHardLinkCopyDirectoryWrapper extends BaseDirectoryTestCase {
@Override
protected Directory getDirectory(Path file) throws IOException {
Directory open = random().nextBoolean() ? newFSDirectory(file) : newDirectory();
return new HardlinkCopyDirectoryWrapper(open);
}
/**
* Tests that we use hardlinks if possible on Directory#copyFrom
*/
public void testCopyHardLinks() throws IOException {
Path tempDir = createTempDir();
Path dir_1 = tempDir.resolve("dir_1");
Path dir_2 = tempDir.resolve("dir_2");
Files.createDirectories(dir_1);
Files.createDirectories(dir_2);
Directory luceneDir_1 = newFSDirectory(dir_1);
Directory luceneDir_2 = newFSDirectory(dir_2);
try {
try (IndexOutput output = luceneDir_1.createOutput("foo.bar", IOContext.DEFAULT)) {
CodecUtil.writeHeader(output, "foo", 0);
output.writeString("hey man, nice shot!");
CodecUtil.writeFooter(output);
}
try {
Files.createLink(tempDir.resolve("test"), dir_1.resolve("foo.bar"));
BasicFileAttributes destAttr = Files.readAttributes(tempDir.resolve("test"), BasicFileAttributes.class);
BasicFileAttributes sourceAttr = Files.readAttributes(dir_1.resolve("foo.bar"), BasicFileAttributes.class);
assumeTrue("hardlinks are not supported", destAttr.fileKey() != null
&& destAttr.fileKey().equals(sourceAttr.fileKey()));
} catch (UnsupportedOperationException ex) {
assumeFalse("hardlinks are not supported", true);
}
HardlinkCopyDirectoryWrapper wrapper = new HardlinkCopyDirectoryWrapper(luceneDir_2);
wrapper.copyFrom(luceneDir_1, "foo.bar", "bar.foo", IOContext.DEFAULT);
assertTrue(Files.exists(dir_2.resolve("bar.foo")));
BasicFileAttributes destAttr = Files.readAttributes(dir_2.resolve("bar.foo"), BasicFileAttributes.class);
BasicFileAttributes sourceAttr = Files.readAttributes(dir_1.resolve("foo.bar"), BasicFileAttributes.class);
assertEquals(destAttr.fileKey(), sourceAttr.fileKey());
try (ChecksumIndexInput indexInput = wrapper.openChecksumInput("bar.foo", IOContext.DEFAULT)) {
CodecUtil.checkHeader(indexInput, "foo", 0, 0);
assertEquals("hey man, nice shot!", indexInput.readString());
CodecUtil.checkFooter(indexInput);
}
} finally {
// close them in a finally block we might run into an assume here
IOUtils.close(luceneDir_1, luceneDir_2);
}
}
}

13
lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointBoxQuery.java
View File

@ -34,6 +34,7 @@ import org.apache.lucene.search.Weight;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.DocIdSetBuilder;
import org.apache.lucene.util.StringHelper;
/**
@ -103,19 +104,21 @@ abstract class LatLonPointBoxQuery extends Query {
return new ConstantScoreWeight(this) {
private DocIdSetIterator buildMatchingIterator(LeafReader reader, PointValues values) throws IOException {
MatchingPoints result = new MatchingPoints(reader, field);
DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
values.intersect(field,
new IntersectVisitor() {
DocIdSetBuilder.BulkAdder adder;
@Override
public void grow(int count) {
result.grow(count);
adder = result.grow(count);
}
@Override
public void visit(int docID) {
result.add(docID);
adder.add(docID);
}
@Override
@ -133,7 +136,7 @@ abstract class LatLonPointBoxQuery extends Query {
}
// Doc is in-bounds
result.add(docID);
adder.add(docID);
}
@Override
@ -160,7 +163,7 @@ abstract class LatLonPointBoxQuery extends Query {
}
}
});
return result.iterator();
return result.build().iterator();
}
@Override

14
lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
View File

@ -32,6 +32,7 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.DocIdSetBuilder;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.SloppyMath;
import org.apache.lucene.util.StringHelper;
@ -117,18 +118,21 @@ final class LatLonPointDistanceQuery extends Query {
LatLonPoint.checkCompatible(fieldInfo);
// matching docids
MatchingPoints result = new MatchingPoints(reader, field);
DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
values.intersect(field,
new IntersectVisitor() {
DocIdSetBuilder.BulkAdder adder;
@Override
public void grow(int count) {
result.grow(count);
adder = result.grow(count);
}
@Override
public void visit(int docID) {
result.add(docID);
adder.add(docID);
}
@Override
@ -152,7 +156,7 @@ final class LatLonPointDistanceQuery extends Query {
// its a match only if its sortKey <= our sortKey
if (SloppyMath.haversinSortKey(latitude, longitude, docLatitude, docLongitude) <= sortKey) {
result.add(docID);
adder.add(docID);
}
}
@ -206,7 +210,7 @@ final class LatLonPointDistanceQuery extends Query {
}
});
return new ConstantScoreScorer(this, score(), result.iterator());
return new ConstantScoreScorer(this, score(), result.build().iterator());
}
};
}

14
lucene/sandbox/src/java/org/apache/lucene/document/LatLonPointInPolygonQuery.java
View File

@ -32,6 +32,7 @@ import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.DocIdSetBuilder;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.geo.Polygon;
@ -110,25 +111,28 @@ final class LatLonPointInPolygonQuery extends Query {
LatLonPoint.checkCompatible(fieldInfo);
// matching docids
MatchingPoints result = new MatchingPoints(reader, field);
DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
values.intersect(field,
new IntersectVisitor() {
DocIdSetBuilder.BulkAdder adder;
@Override
public void grow(int count) {
result.grow(count);
adder = result.grow(count);
}
@Override
public void visit(int docID) {
result.add(docID);
adder.add(docID);
}
@Override
public void visit(int docID, byte[] packedValue) {
if (tree.contains(decodeLatitude(packedValue, 0),
decodeLongitude(packedValue, Integer.BYTES))) {
result.add(docID);
adder.add(docID);
}
}
@ -151,7 +155,7 @@ final class LatLonPointInPolygonQuery extends Query {
}
});
return new ConstantScoreScorer(this, score(), result.iterator());
return new ConstantScoreScorer(this, score(), result.build().iterator());
}
};
}

102
lucene/sandbox/src/java/org/apache/lucene/document/MatchingPoints.java
View File

@ -1,102 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.document;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.SparseFixedBitSet;
/**
* Accumulates matching hits for points.
* <p>
* Add matches with ({@link #add(int)}) and call {@link #iterator()} for
* an iterator over the results.
* <p>
* <b>NOTE:</b> it is required that you implement the optional {@code grow()}
* method in your IntersectVisitor, this is used for cost computation.
* <p>
* This implementation currently optimizes bitset structure (sparse vs dense)
* and {@link DocIdSetIterator#cost()} (cardinality) based on index statistics.
* This API may change as point values evolves.
*
* @lucene.experimental
*/
final class MatchingPoints {
/** bitset we collect into */
private final BitSet bits;
/** number of documents containing a value for the points field */
private final int docCount;
/** number of values indexed for the points field */
private final long numPoints;
/** number of documents in the index segment */
private final int maxDoc;
/** counter of hits seen */
private long counter;
/**
* Creates a new accumulator.
* @param reader reader to collect point matches from
* @param field field name.
*/
public MatchingPoints(LeafReader reader, String field) {
maxDoc = reader.maxDoc();
PointValues values = reader.getPointValues();
if (values == null) {
throw new IllegalStateException("the query is missing null checks");
}
docCount = values.getDocCount(field);
numPoints = values.size(field);
// heuristic: if the field is really sparse, use a sparse impl
if (docCount >= 0 && docCount * 100L < maxDoc) {
bits = new SparseFixedBitSet(maxDoc);
} else {
bits = new FixedBitSet(maxDoc);
}
}
/**
* Record a matching docid.
* <p>
* NOTE: doc IDs do not need to be provided in any order.
*/
public void add(int doc) {
bits.set(doc);
}
/**
* Grows cardinality counter by the given amount.
*/
public void grow(int amount) {
counter += amount;
}
/**
* Returns an iterator over the recorded matches.
*/
public DocIdSetIterator iterator() {
// ensure caller implements the grow() api
assert counter > 0 || bits.cardinality() == 0 : "the IntersectVisitor is missing grow()";
// if single-valued (docCount == numPoints), then we know 1 point == 1 doc
// otherwise we approximate based on field stats
return new BitSetIterator(bits, (long) (counter * (docCount / (double) numPoints)));
}
}

13
lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java
View File

@ -245,19 +245,18 @@ public class TermAutomatonQuery extends Query {
}
private boolean equalsTo(TermAutomatonQuery other) {
// NOTE: not quite correct, because if terms were added in different
// order in each query but the language is the same, we return false:
return checkFinished(this) &&
checkFinished(other) &&
termToID.equals(other.termToID) &&
Operations.sameLanguage(det, other.det);
other == this;
}
@Override
public int hashCode() {
checkFinished(this);
// TODO: LUCENE-7295: Automaton.toDot() is very costly!
return classHash() ^ termToID.hashCode() + det.toDot().hashCode();
// LUCENE-7295: this used to be very awkward toDot() call; it is safer to assume
// that no two instances are equivalent instead (until somebody finds a better way to check
// on automaton equivalence quickly).
return System.identityHashCode(this);
}
/** Returns the dot (graphviz) representation of this automaton.
@ -328,7 +327,6 @@ public class TermAutomatonQuery extends Query {
}
final class TermAutomatonWeight extends Weight {
private final IndexSearcher searcher;
final Automaton automaton;
private final Map<Integer,TermContext> termStates;
private final Similarity.SimWeight stats;
@ -337,7 +335,6 @@ public class TermAutomatonQuery extends Query {
public TermAutomatonWeight(Automaton automaton, IndexSearcher searcher, Map<Integer,TermContext> termStates) throws IOException {
super(TermAutomatonQuery.this);
this.automaton = automaton;
this.searcher = searcher;
this.termStates = termStates;
this.similarity = searcher.getSimilarity(true);
List<TermStatistics> allTermStats = new ArrayList<>();

5
lucene/test-framework/src/java/org/apache/lucene/search/CheckHits.java
View File

@ -346,6 +346,10 @@ public class CheckHits {
if (expl.getDescription().endsWith("computed from:")) {
return; // something more complicated.
}
String descr = expl.getDescription().toLowerCase(Locale.ROOT);
if (descr.startsWith("score based on ") && descr.contains("child docs in range")) {
Assert.assertTrue("Child doc explanations are missing", detail.length > 0);
}
if (detail.length > 0) {
if (detail.length==1) {
// simple containment, unless it's a freq of: (which lets a query explain how the freq is calculated),
@ -357,7 +361,6 @@ public class CheckHits {
// - end with one of: "product of:", "sum of:", "max of:", or
// - have "max plus <x> times others" (where <x> is float).
float x = 0;
String descr = expl.getDescription().toLowerCase(Locale.ROOT);
boolean productOf = descr.endsWith("product of:");
boolean sumOf = descr.endsWith("sum of:");
boolean maxOf = descr.endsWith("max of:");

10
lucene/test-framework/src/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java
View File

@ -17,6 +17,7 @@
package org.apache.lucene.util.automaton;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@ -205,8 +206,8 @@ public class AutomatonTestUtil {
}
public int[] getRandomAcceptedString(Random r) {
final List<Integer> soFar = new ArrayList<>();
int[] codePoints = new int[0];
int codepointCount = 0;
int s = 0;
@ -248,11 +249,12 @@ public class AutomatonTestUtil {
} else {
t = transitions[s][r.nextInt(transitions[s].length)];
}
soFar.add(getRandomCodePoint(r, t.min, t.max));
codePoints = ArrayUtil.grow(codePoints, codepointCount + 1);
codePoints[codepointCount++] = getRandomCodePoint(r, t.min, t.max);
s = t.dest;
}
return ArrayUtil.toIntArray(soFar);
return Arrays.copyOf(codePoints, codepointCount);
}
}

4
lucene/tools/junit4/tests.policy
View File

@ -38,7 +38,9 @@ grant {
permission java.io.FilePermission "${junit4.childvm.cwd}${/}jacoco.db", "write";
permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete";
permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete";
// misc HardlinkCopyDirectoryWrapper needs this to test if hardlinks can be created
permission java.nio.file.LinkPermission "hard";
// needed by SSD detection tests in TestIOUtils (creates symlinks)
permission java.nio.file.LinkPermission "symbolic";

7
solr/CHANGES.txt
View File

@ -236,6 +236,9 @@ Bug Fixes
* SOLR-9151: Fix SolrCLI so that bin/solr -e cloud example can be run from any CWD (janhoy)
* SOLR-9141: Fix ClassCastException when using the /sql handler count() function with
single-shard collections (Minoru Osuka via James Dyer)
Optimizations
----------------------
* SOLR-8722: Don't force a full ZkStateReader refresh on every Overseer operation.
@ -325,6 +328,10 @@ Other Changes
* SOLR-9131: Fix "start solr" text in cluster.vm Velocity template (janhoy)
* SOLR-9159: New cloud based concurrent atomic update test (hossman)
* SOLR-9119: several static methods in ValueSourceParser have been made private (hossman)
================== 6.0.1 ==================
(No Changes)

4
solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
View File

@ -36,12 +36,12 @@ import org.apache.lucene.search.DocValuesRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.uninverting.UninvertingReader.Type;
import com.ibm.icu.text.Collator;
import com.ibm.icu.text.RuleBasedCollator;

24
solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
View File

@ -26,16 +26,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import com.carrotsearch.hppc.IntHashSet;
import com.carrotsearch.hppc.IntObjectHashMap;
import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.LongObjectHashMap;
import com.carrotsearch.hppc.LongObjectMap;
import com.carrotsearch.hppc.cursors.IntObjectCursor;
import com.carrotsearch.hppc.cursors.LongCursor;
import com.carrotsearch.hppc.cursors.LongObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
@ -53,7 +43,6 @@ import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Query;
import org.apache.solr.search.QueryWrapperFilter;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
@ -61,7 +50,6 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@ -87,12 +75,24 @@ import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.DocSlice;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QueryWrapperFilter;
import org.apache.solr.search.SolrConstantScoreQuery;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SortSpecParsing;
import org.apache.solr.uninverting.UninvertingReader;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
import com.carrotsearch.hppc.IntHashSet;
import com.carrotsearch.hppc.IntObjectHashMap;
import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.LongObjectHashMap;
import com.carrotsearch.hppc.LongObjectMap;
import com.carrotsearch.hppc.cursors.IntObjectCursor;
import com.carrotsearch.hppc.cursors.LongCursor;
import com.carrotsearch.hppc.cursors.LongObjectCursor;
import com.carrotsearch.hppc.cursors.ObjectCursor;
/**
* The ExpandComponent is designed to work with the CollapsingPostFilter.
* The CollapsingPostFilter collapses a result set on a field.

24
lucene/misc/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java → solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
View File

@ -14,16 +14,36 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.index;
package org.apache.solr.index;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.CompositeReader;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader.CoreClosedListener;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.search.Sort;
import org.apache.lucene.util.Bits;

2
solr/core/src/java/org/apache/solr/index/SortingMergePolicy.java
View File

@ -33,13 +33,13 @@ import org.apache.lucene.index.SegmentCommitInfo;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.search.Sort;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedLongValues;
import org.apache.solr.index.SlowCompositeReaderWrapper;
// TODO: remove this and add indexSort specification directly to solrconfig.xml? But for BWC, also accept SortingMergePolicy specifiction?

17
solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
View File

@ -28,6 +28,7 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexableField;
@ -41,22 +42,16 @@ import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialArgsParser;
import org.apache.lucene.spatial.query.SpatialOperation;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SpatialOptions;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.apache.solr.util.DistanceUnits;
import org.apache.solr.util.MapListener;
import org.apache.solr.util.SpatialUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import org.locationtech.spatial4j.context.SpatialContext;
import org.locationtech.spatial4j.context.SpatialContextFactory;
import org.locationtech.spatial4j.distance.DistanceUtils;
@ -66,6 +61,12 @@ import org.locationtech.spatial4j.io.SupportedFormats;
import org.locationtech.spatial4j.shape.Point;
import org.locationtech.spatial4j.shape.Rectangle;
import org.locationtech.spatial4j.shape.Shape;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
/**
* Abstract base class for Solr FieldTypes based on a Lucene 4 {@link SpatialStrategy}.

2
solr/core/src/java/org/apache/solr/schema/BinaryField.java
View File

@ -23,10 +23,10 @@ import java.nio.ByteBuffer;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.SortField;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.util.Base64;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

6
solr/core/src/java/org/apache/solr/schema/BoolField.java
View File

@ -22,15 +22,14 @@ import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.BoolDocValues;
import org.apache.lucene.search.SortField;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.CharsRefBuilder;
@ -40,6 +39,7 @@ import org.apache.solr.analysis.SolrAnalyzer;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.function.OrdFieldSource;
import org.apache.solr.uninverting.UninvertingReader.Type;
/**
*
*/

4
solr/core/src/java/org/apache/solr/schema/CollationField.java
View File

@ -40,12 +40,12 @@ import org.apache.lucene.search.DocValuesRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.uninverting.UninvertingReader.Type;
/**
* Field for collated sort keys.

2
solr/core/src/java/org/apache/solr/schema/CurrencyField.java
View File

@ -44,7 +44,7 @@ import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.FieldValueQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.response.TextResponseWriter;

14
solr/core/src/java/org/apache/solr/schema/EnumField.java
View File

@ -16,12 +16,6 @@
*/
package org.apache.solr.schema;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
@ -31,6 +25,12 @@ import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.LegacyIntField;
@ -45,7 +45,6 @@ import org.apache.lucene.search.DocValuesRangeQuery;
import org.apache.lucene.search.LegacyNumericRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.CharsRef;
@ -55,6 +54,7 @@ import org.apache.solr.common.EnumFieldValue;
import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;

8
solr/core/src/java/org/apache/solr/schema/ExternalFileField.java
View File

@ -16,17 +16,17 @@
*/
package org.apache.solr.schema;
import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.function.FileFloatSource;
import java.io.IOException;
import java.util.Map;
import org.apache.solr.uninverting.UninvertingReader.Type;
/** Get values from an external file instead of the index.
*

2
solr/core/src/java/org/apache/solr/schema/FieldType.java
View File

@ -49,7 +49,6 @@ import org.apache.lucene.search.SortedNumericSelector;
import org.apache.lucene.search.SortedSetSelector;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.CharsRef;
@ -66,6 +65,7 @@ import org.apache.solr.query.SolrRangeQuery;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.Sorting;
import org.apache.solr.uninverting.UninvertingReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

10
solr/core/src/java/org/apache/solr/schema/GeoHashField.java
View File

@ -18,23 +18,23 @@ package org.apache.solr.schema;
import java.io.IOException;
import org.locationtech.spatial4j.context.SpatialContext;
import org.locationtech.spatial4j.distance.DistanceUtils;
import org.locationtech.spatial4j.io.GeohashUtils;
import org.locationtech.spatial4j.shape.Point;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.LiteralValueSource;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SolrConstantScoreQuery;
import org.apache.solr.search.SpatialOptions;
import org.apache.solr.search.function.ValueSourceRangeFilter;
import org.apache.solr.search.function.distance.GeohashHaversineFunction;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.apache.solr.util.SpatialUtils;
import org.locationtech.spatial4j.context.SpatialContext;
import org.locationtech.spatial4j.distance.DistanceUtils;
import org.locationtech.spatial4j.io.GeohashUtils;
import org.locationtech.spatial4j.shape.Point;
/**
* This is a class that represents a <a

2
solr/core/src/java/org/apache/solr/schema/IndexSchema.java
View File

@ -51,7 +51,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.solr.uninverting.UninvertingReader;
import org.apache.lucene.util.Version;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;

5
solr/core/src/java/org/apache/solr/schema/LatLonType.java
View File

@ -22,8 +22,8 @@ import java.util.Map;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.VectorValueSource;
@ -37,7 +37,6 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.Weight;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.DelegatingCollector;
@ -45,8 +44,8 @@ import org.apache.solr.search.ExtendedQueryBase;
import org.apache.solr.search.PostFilter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SpatialOptions;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.apache.solr.util.SpatialUtils;
import org.locationtech.spatial4j.context.SpatialContext;
import org.locationtech.spatial4j.distance.DistanceUtils;
import org.locationtech.spatial4j.shape.Point;

4
solr/core/src/java/org/apache/solr/schema/PointType.java
View File

@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.locationtech.spatial4j.distance.DistanceUtils;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
@ -30,13 +29,14 @@ import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SpatialOptions;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.locationtech.spatial4j.distance.DistanceUtils;
/**
* A point type that indexes a point in an n-dimensional space as separate fields and supports range queries.

4
solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
View File

@ -33,14 +33,14 @@ import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.AttributeSource.State;
import org.apache.lucene.util.AttributeSource;
import org.apache.solr.analysis.SolrAnalyzer;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.Sorting;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

4
solr/core/src/java/org/apache/solr/schema/RandomSortField.java
View File

@ -20,16 +20,16 @@ import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
import org.apache.lucene.search.*;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.uninverting.UninvertingReader.Type;
/**
* Utility Field used for random sorting. It should not be passed a value.

2
solr/core/src/java/org/apache/solr/schema/StrField.java
View File

@ -27,10 +27,10 @@ import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.uninverting.UninvertingReader.Type;
public class StrField extends PrimitiveFieldType {

14
solr/core/src/java/org/apache/solr/schema/TextField.java
View File

@ -16,14 +16,16 @@
*/
package org.apache.solr.schema;
import java.io.IOException;
import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.SortedSetFieldSource;
import org.apache.lucene.search.*;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.QueryBuilder;
import org.apache.solr.common.SolrException;
@ -31,9 +33,7 @@ import org.apache.solr.query.SolrRangeQuery;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.Sorting;
import java.util.Map;
import java.io.IOException;
import org.apache.solr.uninverting.UninvertingReader.Type;
/** <code>TextField</code> is the basic type for configurable text analysis.
* Analyzers for field types using this implementation should be defined in the schema.

4
solr/core/src/java/org/apache/solr/schema/TrieField.java
View File

@ -26,8 +26,8 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.FieldType.LegacyNumericType;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.LegacyDoubleField;
import org.apache.lucene.document.LegacyFloatField;
import org.apache.lucene.document.LegacyIntField;
@ -47,7 +47,6 @@ import org.apache.lucene.search.LegacyNumericRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedSetSelector;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.CharsRef;
@ -61,6 +60,7 @@ import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.FunctionRangeQuery;
import org.apache.solr.search.QParser;
import org.apache.solr.search.function.ValueSourceRangeFilter;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.apache.solr.util.DateMathParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

21
solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
View File

@ -25,15 +25,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import com.carrotsearch.hppc.FloatArrayList;
import com.carrotsearch.hppc.IntArrayList;
import com.carrotsearch.hppc.IntIntHashMap;
import com.carrotsearch.hppc.IntLongHashMap;
import com.carrotsearch.hppc.cursors.IntIntCursor;
import com.carrotsearch.hppc.cursors.IntLongCursor;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
@ -49,13 +41,12 @@ import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.BytesRef;
@ -65,8 +56,8 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.handler.component.QueryElevationComponent;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
@ -75,6 +66,14 @@ import org.apache.solr.schema.StrField;
import org.apache.solr.schema.TrieFloatField;
import org.apache.solr.schema.TrieIntField;
import org.apache.solr.schema.TrieLongField;
import org.apache.solr.uninverting.UninvertingReader;
import com.carrotsearch.hppc.FloatArrayList;
import com.carrotsearch.hppc.IntArrayList;
import com.carrotsearch.hppc.IntIntHashMap;
import com.carrotsearch.hppc.IntLongHashMap;
import com.carrotsearch.hppc.cursors.IntIntCursor;
import com.carrotsearch.hppc.cursors.IntLongCursor;
/**

6
solr/core/src/java/org/apache/solr/search/Insanity.java
View File

@ -20,16 +20,16 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.solr.uninverting.UninvertingReader;
/**
* Lucene 5.0 removes "accidental" insanity, so you must explicitly

5
solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java
View File

@ -18,13 +18,12 @@ package org.apache.solr.search;
import java.net.URL;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.JmxMonitoredMap.JmxAugmentedSolrInfoMBean;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean;
import org.apache.solr.core.JmxMonitoredMap.JmxAugmentedSolrInfoMBean;
import org.apache.solr.uninverting.UninvertingReader;
/**
* A SolrInfoMBean that provides introspection of the Solr FieldCache

4
solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -53,7 +53,6 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiPostingsEnum;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
@ -94,7 +93,6 @@ import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.TotalHitCountCollector;
import org.apache.lucene.search.Weight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
@ -109,6 +107,7 @@ import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean;
import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
@ -122,6 +121,7 @@ import org.apache.solr.schema.TrieFloatField;
import org.apache.solr.schema.TrieIntField;
import org.apache.solr.search.facet.UnInvertedField;
import org.apache.solr.search.stats.StatsSource;
import org.apache.solr.uninverting.UninvertingReader;
import org.apache.solr.update.IndexFingerprint;
import org.apache.solr.update.SolrIndexConfig;
import org.slf4j.Logger;

18
solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
View File

@ -91,25 +91,29 @@ public abstract class ValueSourceParser implements NamedListInitializedPlugin {
*/
public abstract ValueSource parse(FunctionQParser fp) throws SyntaxError;
/* standard functions */
public static Map<String, ValueSourceParser> standardValueSourceParsers = new HashMap<>();
/** standard functions supported by default, filled in static class initialization */
private static final Map<String, ValueSourceParser> standardVSParsers = new HashMap<>();
/** standard functions supported by default */
public static final Map<String, ValueSourceParser> standardValueSourceParsers
= Collections.unmodifiableMap(standardVSParsers);
/** Adds a new parser for the name and returns any existing one that was overridden.
* This is not thread safe.
*/
public static ValueSourceParser addParser(String name, ValueSourceParser p) {
return standardValueSourceParsers.put(name, p);
private static ValueSourceParser addParser(String name, ValueSourceParser p) {
return standardVSParsers.put(name, p);
}
/** Adds a new parser for the name and returns any existing one that was overridden.
* This is not thread safe.
*/
public static ValueSourceParser addParser(NamedParser p) {
return standardValueSourceParsers.put(p.name(), p);
private static ValueSourceParser addParser(NamedParser p) {
return standardVSParsers.put(p.name(), p);
}
private static void alias(String source, String dest) {
standardValueSourceParsers.put(dest, standardValueSourceParsers.get(source));
standardVSParsers.put(dest, standardVSParsers.get(source));
}
static {

4
solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java
View File

@ -27,17 +27,16 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.uninverting.DocTermOrds;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.solr.common.SolrException;
import org.apache.solr.core.SolrCore;
import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.TrieField;
import org.apache.solr.search.BitDocSet;
@ -45,6 +44,7 @@ import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SolrCache;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.uninverting.DocTermOrds;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

6
solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java
View File

@ -20,13 +20,12 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@ -34,6 +33,7 @@ import org.apache.lucene.queries.function.docvalues.IntDocValues;
import org.apache.lucene.search.SortedSetSelector;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueInt;
import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.Insanity;
import org.apache.solr.search.SolrIndexSearcher;

6
solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java
View File

@ -20,18 +20,18 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
import org.apache.lucene.search.SortedSetSelector;
import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.Insanity;
import org.apache.solr.search.SolrIndexSearcher;

2
solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
View File

@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.join.JoinUtil;
import org.apache.lucene.search.join.ScoreMode;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Aliases;
@ -45,6 +44,7 @@ import org.apache.solr.search.QParser;
import org.apache.solr.search.QParserPlugin;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SyntaxError;
import org.apache.solr.uninverting.UninvertingReader;
import org.apache.solr.util.RefCounted;
/**

15
lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java → solr/core/src/java/org/apache/solr/uninverting/DocTermOrds.java
View File

@ -14,19 +14,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.codecs.PostingsFormat; // javadocs
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@ -276,7 +277,7 @@ public class DocTermOrds implements Accountable {
throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
}
//System.out.println("DTO uninvert field=" + field + " prefix=" + termPrefix);
final long startTime = System.currentTimeMillis();
final long startTime = System.nanoTime();
prefix = termPrefix == null ? null : BytesRef.deepCopyOf(termPrefix);
final int maxDoc = reader.maxDoc();
@ -444,7 +445,7 @@ public class DocTermOrds implements Accountable {
numTermsInField = termNum;
long midPoint = System.currentTimeMillis();
long midPoint = System.nanoTime();
if (termInstances == 0) {
// we didn't invert anything
@ -533,10 +534,10 @@ public class DocTermOrds implements Accountable {
}
indexedTermsArray = indexedTerms.toArray(new BytesRef[indexedTerms.size()]);
long endTime = System.currentTimeMillis();
long endTime = System.nanoTime();
total_time = (int)(endTime-startTime);
phase1_time = (int)(midPoint-startTime);
total_time = (int) TimeUnit.MILLISECONDS.convert(endTime-startTime, TimeUnit.NANOSECONDS);
phase1_time = (int) TimeUnit.MILLISECONDS.convert(midPoint-startTime, TimeUnit.NANOSECONDS);
}
/** Number of bytes to represent an unsigned int as a vint. */

2
lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java → solr/core/src/java/org/apache/solr/uninverting/FieldCache.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.io.IOException;
import java.io.PrintStream;

2
lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java → solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.io.IOException;
import java.io.PrintStream;

4
lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheSanityChecker.java → solr/core/src/java/org/apache/solr/uninverting/FieldCacheSanityChecker.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.util.ArrayList;
import java.util.Collection;
@ -27,8 +27,8 @@ import java.util.Set;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.uninverting.FieldCache.CacheEntry;
import org.apache.lucene.util.MapOfSets;
import org.apache.solr.uninverting.FieldCache.CacheEntry;
/**
* Provides methods for sanity checking that entries in the FieldCache

4
lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java → solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.io.IOException;
import java.util.ArrayList;
@ -37,8 +37,8 @@ import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.uninverting.FieldCache.CacheEntry;
import org.apache.lucene.util.Bits;
import org.apache.solr.uninverting.FieldCache.CacheEntry;
/**
* A FilterReader that exposes <i>indexed</i> values as if they also had

2
lucene/misc/src/java/org/apache/lucene/uninverting/package-info.java → solr/core/src/java/org/apache/solr/uninverting/package-info.java
View File

@ -18,4 +18,4 @@
/**
* Support for creating docvalues on-the-fly from the inverted index at runtime.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;

3
solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java
View File

@ -29,8 +29,9 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.uninverting.UninvertingReader;
import org.apache.solr.uninverting.UninvertingReader;
/**
* Allows access to uninverted docvalues by delete-by-queries.

2
solr/core/src/java/org/apache/solr/update/VersionInfo.java
View File

@ -24,7 +24,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@ -34,6 +33,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LegacyNumericUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.SuppressForbidden;
import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;

38
solr/core/src/test-files/solr/collection1/conf/schema-minimal-atomic-stress.xml Normal file
View File

@ -0,0 +1,38 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<schema name="minimal-atomic-stress" version="1.6">
<!-- minimal fields needed to stress test atomic updates
See: TestStressCloudBlindAtomicUpdates
-->
<uniqueKey>id</uniqueKey>
<field name="id" type="string" indexed="true" stored="true" docValues="true"/>
<field name="_version_" type="long" indexed="false" stored="false" docValues="true" />
<!-- atomic updates should work on all of these permutations -->
<field name="long_dv" type="long" indexed="false" stored="false" docValues="true" />
<field name="long_dv_stored" type="long" indexed="false" stored="true" docValues="true" />
<field name="long_dv_stored_idx" type="long" indexed="true" stored="true" docValues="true" />
<field name="long_dv_idx" type="long" indexed="true" stored="false" docValues="true" />
<field name="long_stored_idx" type="long" indexed="true" stored="true" docValues="false" />
<fieldType name="string" class="solr.StrField" multiValued="false" indexed="false" stored="false" docValues="false" />
<fieldType name="long" class="solr.TrieLongField" multiValued="false" indexed="false" stored="false" docValues="false"/>
<!-- unused, but play nice with existing solrconfig so we don't have to create a new one just for this test -->
<dynamicField name="*" type="string" indexed="true" stored="true" />
</schema>

178
solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java
View File

@ -159,182 +159,114 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
protected Set<String> commonMocks(int liveNodesCount) throws Exception {
shardHandlerFactoryMock.getShardHandler();
expectLastCall().andAnswer(new IAnswer<ShardHandler>() {
@Override
public ShardHandler answer() throws Throwable {
log.info("SHARDHANDLER");
return shardHandlerMock;
}
expectLastCall().andAnswer(() -> {
log.info("SHARDHANDLER");
return shardHandlerMock;
}).anyTimes();
workQueueMock.peekTopN(EasyMock.anyInt(), anyObject(Set.class), EasyMock.anyLong());
expectLastCall().andAnswer(new IAnswer<List>() {
@Override
public List answer() throws Throwable {
Object result;
int count = 0;
while ((result = queue.peek()) == null) {
Thread.sleep(1000);
count++;
if (count > 1) return null;
}
return Arrays.asList(result);
expectLastCall().andAnswer(() -> {
Object result;
int count = 0;
while ((result = queue.peek()) == null) {
Thread.sleep(1000);
count++;
if (count > 1) return null;
}
return Arrays.asList(result);
}).anyTimes();
workQueueMock.getTailId();
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
Object result = null;
Iterator iter = queue.iterator();
while(iter.hasNext()) {
result = iter.next();
}
return result==null ? null : ((QueueEvent)result).getId();
expectLastCall().andAnswer(() -> {
Object result = null;
Iterator iter = queue.iterator();
while(iter.hasNext()) {
result = iter.next();
}
return result==null ? null : ((QueueEvent)result).getId();
}).anyTimes();
workQueueMock.peek(true);
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
Object result;
while ((result = queue.peek()) == null) {
Thread.sleep(1000);
}
return result;
expectLastCall().andAnswer(() -> {
Object result;
while ((result = queue.peek()) == null) {
Thread.sleep(1000);
}
return result;
}).anyTimes();
workQueueMock.remove(anyObject(QueueEvent.class));
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
queue.remove((QueueEvent) getCurrentArguments()[0]);
return null;
}
expectLastCall().andAnswer(() -> {
queue.remove(getCurrentArguments()[0]);
return null;
}).anyTimes();
workQueueMock.poll();
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return queue.poll();
}
}).anyTimes();
expectLastCall().andAnswer(() -> queue.poll()).anyTimes();
zkStateReaderMock.getClusterState();
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return clusterStateMock;
}
}).anyTimes();
expectLastCall().andAnswer(() -> clusterStateMock).anyTimes();
zkStateReaderMock.getZkClient();
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return solrZkClientMock;
}
}).anyTimes();
expectLastCall().andAnswer(() -> solrZkClientMock).anyTimes();
zkStateReaderMock.updateClusterState();
clusterStateMock.getCollections();
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return collectionsSet;
}
}).anyTimes();
expectLastCall().andAnswer(() -> collectionsSet).anyTimes();
final Set<String> liveNodes = new HashSet<>();
for (int i = 0; i < liveNodesCount; i++) {
final String address = "localhost:" + (8963 + i) + "_solr";
liveNodes.add(address);
zkStateReaderMock.getBaseUrlForNodeName(address);
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
// This works as long as this test does not use a
// webapp context with an underscore in it
return address.replaceAll("_", "/");
}
expectLastCall().andAnswer(() -> {
// This works as long as this test does not use a
// webapp context with an underscore in it
return address.replaceAll("_", "/");
}).anyTimes();
}
zkStateReaderMock.getClusterProperty("legacyCloud", "true");
expectLastCall().andAnswer(new IAnswer<String>() {
@Override
public String answer() throws Throwable {
return "true";
}
});
expectLastCall().andAnswer(() -> "true");
solrZkClientMock.getZkClientTimeout();
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return 30000;
}
}).anyTimes();
expectLastCall().andAnswer(() -> 30000).anyTimes();
clusterStateMock.hasCollection(anyObject(String.class));
expectLastCall().andAnswer(new IAnswer<Boolean>() {
@Override
public Boolean answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
return collectionsSet.contains(key);
}
} ).anyTimes();
expectLastCall().andAnswer(() -> {
String key = (String) getCurrentArguments()[0];
return collectionsSet.contains(key);
}).anyTimes();
clusterStateMock.getLiveNodes();
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return liveNodes;
}
}).anyTimes();
expectLastCall().andAnswer(() -> liveNodes).anyTimes();
solrZkClientMock.create(anyObject(String.class), anyObject(byte[].class), anyObject(CreateMode.class), anyBoolean());
expectLastCall().andAnswer(new IAnswer<String>() {
@Override
public String answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
zkMap.put(key, null);
handleCreateCollMessage((byte[]) getCurrentArguments()[1]);
return key;
}
expectLastCall().andAnswer(() -> {
String key = (String) getCurrentArguments()[0];
zkMap.put(key, null);
handleCreateCollMessage((byte[]) getCurrentArguments()[1]);
return key;
}).anyTimes();
solrZkClientMock.makePath(anyObject(String.class), anyObject(byte[].class), anyBoolean());
expectLastCall().andAnswer(new IAnswer<String>() {
@Override
public String answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
return key;
}
expectLastCall().andAnswer(() -> {
String key = (String) getCurrentArguments()[0];
return key;
}).anyTimes();
solrZkClientMock.makePath(anyObject(String.class), anyObject(byte[].class), anyObject(CreateMode.class), anyBoolean());
expectLastCall().andAnswer(new IAnswer<String>() {
@Override
public String answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
return key;
}
expectLastCall().andAnswer(() -> {
String key = (String) getCurrentArguments()[0];
return key;
}).anyTimes();
solrZkClientMock.exists(anyObject(String.class),anyBoolean());
expectLastCall().andAnswer(new IAnswer<Boolean>() {
@Override
public Boolean answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
return zkMap.containsKey(key);
}
expectLastCall().andAnswer(() -> {
String key = (String) getCurrentArguments()[0];
return zkMap.containsKey(key);
}).anyTimes();
zkMap.put("/configs/myconfig", null);

483
solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java Normal file
View File

@ -0,0 +1,483 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import java.io.File;
import java.lang.invoke.MethodHandles;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.client.solrj.request.schema.SchemaRequest.Field;
import org.apache.solr.client.solrj.request.schema.SchemaRequest.FieldType;
import org.apache.solr.client.solrj.response.schema.SchemaResponse.FieldResponse;
import org.apache.solr.client.solrj.response.schema.SchemaResponse.FieldTypeResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.TestInjection;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.TestUtil;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Stress test of Atomic Updates in a MinCloud Cluster.
*
* Focus of test is parallel threads hammering updates on diff docs using random clients/nodes,
* Optimistic Concurrency is not used here because of SOLR-8733, instead we just throw lots of
* "inc" operations at a numeric field and check that the math works out at the end.
*/
@Slow
@SuppressSSL(bugUrl="SSL overhead seems to cause OutOfMemory when stress testing")
public class TestStressCloudBlindAtomicUpdates extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();
private static final String COLLECTION_NAME = "test_col";
/** A basic client for operations at the cloud level, default collection will be set */
private static CloudSolrClient CLOUD_CLIENT;
/** One client per node */
private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
/** Service to execute all parallel work
* @see #NUM_THREADS
*/
private static ExecutorService EXEC_SERVICE;
/** num parallel threads in use by {@link #EXEC_SERVICE} */
private static int NUM_THREADS;
/**
* Used as an increment and multiplier when deciding how many docs should be in
* the test index. 1 means every doc in the index is a candidate for updates, bigger numbers mean a
* larger index is used (so tested docs are more likeely to be spread out in multiple segments)
*/
private static int DOC_ID_INCR;
@BeforeClass
private static void createMiniSolrCloudCluster() throws Exception {
// NOTE: numDocsToCheck uses atLeast, so nightly & multiplier are alreayd a factor in index size
// no need to redundently factor them in here as well
DOC_ID_INCR = TestUtil.nextInt(random(), 1, 7);
NUM_THREADS = atLeast(3);
EXEC_SERVICE = ExecutorUtil.newMDCAwareFixedThreadPool
(NUM_THREADS, new DefaultSolrThreadFactory(DEBUG_LABEL));
// at least 2, but don't go crazy on nightly/test.multiplier with "atLeast()"
final int numShards = TEST_NIGHTLY ? 5 : 2;
final int repFactor = 2;
final int numNodes = numShards * repFactor;
final String configName = DEBUG_LABEL + "_config-set";
final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
configureCluster(numNodes).addConfig(configName, configDir).configure();
Map<String, String> collectionProperties = new HashMap<>();
collectionProperties.put("config", "solrconfig-tlog.xml");
collectionProperties.put("schema", "schema-minimal-atomic-stress.xml");
assertNotNull(cluster.createCollection(COLLECTION_NAME, numShards, repFactor,
configName, null, null, collectionProperties));
CLOUD_CLIENT = cluster.getSolrClient();
CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
waitForRecoveriesToFinish(CLOUD_CLIENT);
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
}
// sanity check no one broke the assumptions we make about our schema
checkExpectedSchemaType( map("name","long",
"class","solr.TrieLongField",
"multiValued",Boolean.FALSE,
"indexed",Boolean.FALSE,
"stored",Boolean.FALSE,
"docValues",Boolean.FALSE) );
}
@AfterClass
private static void afterClass() throws Exception {
TestInjection.reset();
ExecutorUtil.shutdownAndAwaitTermination(EXEC_SERVICE);
EXEC_SERVICE = null;
CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
for (HttpSolrClient client : CLIENTS) {
client.close();
}
CLIENTS = null;
}
@Before
private void clearCloudCollection() throws Exception {
assertEquals(0, CLOUD_CLIENT.deleteByQuery("*:*").getStatus());
assertEquals(0, CLOUD_CLIENT.optimize().getStatus());
TestInjection.reset();
final int injectionPercentage = (int)Math.ceil(atLeast(1) / 2);
final String testInjection = usually() ? "false:0" : ("true:" + injectionPercentage);
log.info("TestInjection: fail replica, update pause, tlog pauses: " + testInjection);
TestInjection.failReplicaRequests = testInjection;
TestInjection.updateLogReplayRandomPause = testInjection;
TestInjection.updateRandomPause = testInjection;
}
public void test_dv() throws Exception {
String field = "long_dv";
checkExpectedSchemaField(map("name", field,
"type","long",
"stored",Boolean.FALSE,
"indexed",Boolean.FALSE,
"docValues",Boolean.TRUE));
checkField(field);
}
public void test_dv_stored() throws Exception {
String field = "long_dv_stored";
checkExpectedSchemaField(map("name", field,
"type","long",
"stored",Boolean.TRUE,
"indexed",Boolean.FALSE,
"docValues",Boolean.TRUE));
checkField(field);
}
public void test_dv_stored_idx() throws Exception {
String field = "long_dv_stored_idx";
checkExpectedSchemaField(map("name", field,
"type","long",
"stored",Boolean.TRUE,
"indexed",Boolean.TRUE,
"docValues",Boolean.TRUE));
checkField(field);
}
public void test_dv_idx() throws Exception {
String field = "long_dv_idx";
checkExpectedSchemaField(map("name", field,
"type","long",
"stored",Boolean.FALSE,
"indexed",Boolean.TRUE,
"docValues",Boolean.TRUE));
checkField(field);
}
public void test_stored_idx() throws Exception {
String field = "long_stored_idx";
checkExpectedSchemaField(map("name", field,
"type","long",
"stored",Boolean.TRUE,
"indexed",Boolean.TRUE,
"docValues",Boolean.FALSE));
checkField(field);
}
public void checkField(final String numericFieldName) throws Exception {
final CountDownLatch abortLatch = new CountDownLatch(1);
final int numDocsToCheck = atLeast(37);
final int numDocsInIndex = (numDocsToCheck * DOC_ID_INCR);
final AtomicLong[] expected = new AtomicLong[numDocsToCheck];
log.info("Testing " + numericFieldName + ": numDocsToCheck=" + numDocsToCheck + ", numDocsInIndex=" + numDocsInIndex + ", incr=" + DOC_ID_INCR);
// seed the index & keep track of what docs exist and with what values
for (int id = 0; id < numDocsInIndex; id++) {
// NOTE: the field we're mutating is a long, but we seed with a random int,
// and we will inc/dec by random smaller ints, to ensure we never over/under flow
final int initValue = random().nextInt();
SolrInputDocument doc = doc(f("id",""+id), f(numericFieldName, initValue));
UpdateResponse rsp = update(doc).process(CLOUD_CLIENT);
assertEquals(doc.toString() + " => " + rsp.toString(), 0, rsp.getStatus());
if (0 == id % DOC_ID_INCR) {
expected[(int)(id / DOC_ID_INCR)] = new AtomicLong(initValue);
}
}
assertNotNull("Sanity Check no off-by-one in expected init: ", expected[expected.length-1]);
// sanity check index contents
assertEquals(0, CLOUD_CLIENT.commit().getStatus());
assertEquals(numDocsInIndex,
CLOUD_CLIENT.query(params("q", "*:*")).getResults().getNumFound());
// spin up parallel workers to hammer updates
List<Future<Worker>> results = new ArrayList<Future<Worker>>(NUM_THREADS);
for (int workerId = 0; workerId < NUM_THREADS; workerId++) {
Worker worker = new Worker(workerId, expected, abortLatch, new Random(random().nextLong()),
numericFieldName);
// ask for the Worker to be returned in the Future so we can inspect it
results.add(EXEC_SERVICE.submit(worker, worker));
}
// check the results of all our workers
for (Future<Worker> r : results) {
try {
Worker w = r.get();
if (! w.getFinishedOk() ) {
// quick and dirty sanity check if any workers didn't succeed, but didn't throw an exception either
abortLatch.countDown();
log.error("worker={} didn't finish ok, but didn't throw exception?", w.workerId);
}
} catch (ExecutionException ee) {
Throwable rootCause = ee.getCause();
if (rootCause instanceof Error) {
// low level error, or test assertion failure - either way don't leave it wrapped
log.error("Worker exec Error, throwing root cause", ee);
throw (Error) rootCause;
} else {
log.error("Worker ExecutionException, re-throwing", ee);
throw ee;
}
}
}
assertEquals("Abort latch has changed, why didn't we get an exception from a worker?",
1L, abortLatch.getCount());
TestInjection.reset();
waitForRecoveriesToFinish(CLOUD_CLIENT);
// check all the final index contents match our expectations
int incorrectDocs = 0;
for (int id = 0; id < numDocsInIndex; id += DOC_ID_INCR) {
assert 0 == id % DOC_ID_INCR : "WTF? " + id;
final long expect = expected[(int)(id / DOC_ID_INCR)].longValue();
final String docId = "" + id;
// sometimes include an fq on the expected value to ensure the updated values
// are "visible" for searching
final SolrParams p = (0 != TestUtil.nextInt(random(), 0,15))
? params() : params("fq",numericFieldName + ":" + expect);
SolrDocument doc = getRandClient(random()).getById(docId, p);
final boolean foundWithFilter = (null != doc);
if (! foundWithFilter) {
// try again w/o fq to see what it does have
doc = getRandClient(random()).getById(docId);
}
Long actual = (null == doc) ? null : (Long) doc.getFirstValue(numericFieldName);
if (actual == null || expect != actual.longValue() || ! foundWithFilter) {
log.error("docId={}, foundWithFilter={}, expected={}, actual={}",
docId, foundWithFilter, expect, actual);
incorrectDocs++;
}
}
assertEquals("Some docs had errors -- check logs", 0, incorrectDocs);
}
public static final class Worker implements Runnable {
public final int workerId;
final AtomicLong[] expected;
final CountDownLatch abortLatch;
final Random rand;
final String updateField;
final int numDocsToUpdate;
boolean ok = false; // set to true only on successful completion
public Worker(int workerId, AtomicLong[] expected, CountDownLatch abortLatch, Random rand,
String updateField) {
this.workerId = workerId;
this.expected = expected;
this.abortLatch = abortLatch;
this.rand = rand;
this.updateField = updateField;
this.numDocsToUpdate = atLeast(rand, 25);
}
public boolean getFinishedOk() {
return ok;
}
private void doRandomAtomicUpdate(int docId) throws Exception {
assert 0 == docId % DOC_ID_INCR : "WTF? " + docId;
final int delta = TestUtil.nextInt(rand, -1000, 1000);
log.info("worker={}, docId={}, delta={}", workerId, docId, delta);
SolrClient client = getRandClient(rand);
SolrInputDocument doc = doc(f("id",""+docId),
f(updateField,Collections.singletonMap("inc",delta)));
UpdateResponse rsp = update(doc).process(client);
assertEquals(doc + " => " + rsp, 0, rsp.getStatus());
AtomicLong counter = expected[(int)(docId / DOC_ID_INCR)];
assertNotNull("null counter for " + docId + "/" + DOC_ID_INCR, counter);
counter.getAndAdd(delta);
}
public void run() {
final String origThreadName = Thread.currentThread().getName();
try {
Thread.currentThread().setName(origThreadName + "-w" + workerId);
final int maxDocMultiplier = expected.length-1;
for (int docIter = 0; docIter < numDocsToUpdate; docIter++) {
final int docId = DOC_ID_INCR * TestUtil.nextInt(rand, 0, maxDocMultiplier);
// tweak our thread name to keep track of what we're up to
Thread.currentThread().setName(origThreadName + "-w" + workerId + "-d" + docId);
// no matter how random the doc selection may be per thread, ensure
// every doc that is selected by *a* thread gets at least a couple rapid fire updates
final int itersPerDoc = atLeast(rand, 2);
for (int updateIter = 0; updateIter < itersPerDoc; updateIter++) {
if (0 == abortLatch.getCount()) {
return;
}
doRandomAtomicUpdate(docId);
}
if (rand.nextBoolean()) { Thread.yield(); }
}
} catch (Error err) {
log.error(Thread.currentThread().getName(), err);
abortLatch.countDown();
throw err;
} catch (Exception ex) {
log.error(Thread.currentThread().getName(), ex);
abortLatch.countDown();
throw new RuntimeException(ex.getMessage(), ex);
} finally {
Thread.currentThread().setName(origThreadName);
}
ok = true;
}
}
public static UpdateRequest update(SolrInputDocument... docs) {
return update(null, docs);
}
public static UpdateRequest update(SolrParams params, SolrInputDocument... docs) {
UpdateRequest r = new UpdateRequest();
if (null != params) {
r.setParams(new ModifiableSolrParams(params));
}
r.add(Arrays.asList(docs));
return r;
}
public static SolrInputDocument doc(SolrInputField... fields) {
SolrInputDocument doc = new SolrInputDocument();
for (SolrInputField f : fields) {
doc.put(f.getName(), f);
}
return doc;
}
public static SolrInputField f(String fieldName, Object... values) {
SolrInputField f = new SolrInputField(fieldName);
f.setValue(values, 1.0F);
// TODO: soooooooooo stupid (but currently neccessary because atomic updates freak out
// if the Map with the "inc" operation is inside of a collection - even if it's the only "value") ...
if (1 == values.length) {
f.setValue(values[0], 1.0F);
} else {
f.setValue(values, 1.0F);
}
return f;
}
public static SolrClient getRandClient(Random rand) {
int numClients = CLIENTS.size();
int idx = TestUtil.nextInt(rand, 0, numClients);
return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx);
}
public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception {
assert null != client.getDefaultCollection();
AbstractDistribZkTestBase.waitForRecoveriesToFinish(client.getDefaultCollection(),
client.getZkStateReader(),
true, true, 330);
}
/**
* Use the schema API to verify that the specified expected Field exists with those exact attributes.
* @see #CLOUD_CLIENT
*/
public static void checkExpectedSchemaField(Map<String,Object> expected) throws Exception {
String fieldName = (String) expected.get("name");
assertNotNull("expected contains no name: " + expected, fieldName);
FieldResponse rsp = new Field(fieldName).process(CLOUD_CLIENT);
assertNotNull("Field Null Response: " + fieldName, rsp);
assertEquals("Field Status: " + fieldName + " => " + rsp.toString(), 0, rsp.getStatus());
assertEquals("Field: " + fieldName, expected, rsp.getField());
}
/**
* Use the schema API to verify that the specified expected FieldType exists with those exact attributes.
* @see #CLOUD_CLIENT
*/
public static void checkExpectedSchemaType(Map<String,Object> expected) throws Exception {
String typeName = (String) expected.get("name");
assertNotNull("expected contains no type: " + expected, typeName);
FieldTypeResponse rsp = new FieldType(typeName).process(CLOUD_CLIENT);
assertNotNull("FieldType Null Response: " + typeName, rsp);
assertEquals("FieldType Status: " + typeName + " => " + rsp.toString(), 0, rsp.getStatus());
assertEquals("FieldType: " + typeName, expected, rsp.getFieldType().getAttributes());
}
}

6
lucene/misc/src/test/org/apache/lucene/index/TestSlowCompositeReaderWrapper.java → solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.index;
package org.apache.solr.index;
import java.io.IOException;
import java.util.ArrayList;
@ -23,6 +23,10 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;

2
solr/core/src/test/org/apache/solr/request/TestFaceting.java
View File

@ -25,12 +25,12 @@ import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.uninverting.DocTermOrds;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.uninverting.DocTermOrds;
import org.apache.solr.util.RefCounted;
import org.junit.After;
import org.junit.BeforeClass;

4
solr/core/src/test/org/apache/solr/search/TestSort.java
View File

@ -42,13 +42,12 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortField.Type;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopFieldCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
@ -56,6 +55,7 @@ import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.uninverting.UninvertingReader;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

35
solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.java
View File

@ -100,13 +100,7 @@ public class TestPKIAuthenticationPlugin extends SolrTestCaseJ4 {
assertTrue(header.get().getValue().startsWith(nodeName));
final AtomicReference<ServletRequest> wrappedRequestByFilter = new AtomicReference<>();
HttpServletRequest mockReq = createMockRequest(header);
FilterChain filterChain = new FilterChain() {
@Override
public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse)
throws IOException, ServletException {
wrappedRequestByFilter.set(servletRequest);
}
};
FilterChain filterChain = (servletRequest, servletResponse) -> wrappedRequestByFilter.set(servletRequest);
mock.doAuthenticate(mockReq, null, filterChain);
assertNotNull(wrappedRequestByFilter.get());
@ -164,30 +158,17 @@ public class TestPKIAuthenticationPlugin extends SolrTestCaseJ4 {
HttpServletRequest mockReq = EasyMock.createMock(HttpServletRequest.class);
EasyMock.reset(mockReq);
mockReq.getHeader(EasyMock.anyObject(String.class));
EasyMock.expectLastCall().andAnswer(new IAnswer<String>() {
@Override
public String answer() throws Throwable {
if (PKIAuthenticationPlugin.HEADER.equals(getCurrentArguments()[0])) {
if (header.get() == null) return null;
return header.get().getValue();
} else return null;
}
EasyMock.expectLastCall().andAnswer(() -> {
if (PKIAuthenticationPlugin.HEADER.equals(getCurrentArguments()[0])) {
if (header.get() == null) return null;
return header.get().getValue();
} else return null;
}).anyTimes();
mockReq.getUserPrincipal();
EasyMock.expectLastCall().andAnswer(new IAnswer<Principal>() {
@Override
public Principal answer() throws Throwable {
return null;
}
}).anyTimes();
EasyMock.expectLastCall().andAnswer(() -> null).anyTimes();
mockReq.getRequestURI();
EasyMock.expectLastCall().andAnswer(new IAnswer<String>() {
@Override
public String answer() throws Throwable {
return "/collection1/select";
}
}).anyTimes();
EasyMock.expectLastCall().andAnswer(() -> "/collection1/select").anyTimes();
EasyMock.replay(mockReq);
return mockReq;

16
lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java → solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrds.java
View File

@ -14,13 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
@ -31,28 +31,28 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.LegacyIntField;
import org.apache.lucene.document.LegacyLongField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LegacyNumericUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.index.SlowCompositeReaderWrapper;
// TODO:
// - test w/ del docs

12
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCache.java → solr/core/src/test/org/apache/solr/uninverting/TestFieldCache.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@ -31,11 +31,11 @@ import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.LegacyDoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LegacyDoubleField;
import org.apache.lucene.document.LegacyFloatField;
import org.apache.lucene.document.LegacyIntField;
import org.apache.lucene.document.LegacyLongField;
@ -44,16 +44,15 @@ import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
@ -65,6 +64,7 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LegacyNumericUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.junit.AfterClass;
import org.junit.BeforeClass;

2
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheReopen.java → solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheReopen.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;

12
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSanityChecker.java → solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSanityChecker.java
View File

@ -14,26 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.LegacyDoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LegacyDoubleField;
import org.apache.lucene.document.LegacyFloatField;
import org.apache.lucene.document.LegacyIntField;
import org.apache.lucene.document.LegacyLongField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.uninverting.FieldCacheSanityChecker.Insanity;
import org.apache.lucene.uninverting.FieldCacheSanityChecker.InsanityType;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.apache.solr.uninverting.FieldCacheSanityChecker.Insanity;
import org.apache.solr.uninverting.FieldCacheSanityChecker.InsanityType;
public class TestFieldCacheSanityChecker extends LuceneTestCase {

4
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSort.java → solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSort.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.io.IOException;
import java.util.Collections;
@ -51,7 +51,7 @@ import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;

4
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSortRandom.java → solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheSortRandom.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.io.IOException;
import java.util.ArrayList;
@ -50,12 +50,12 @@ import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.search.Weight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.uninverting.UninvertingReader.Type;
/** random sorting tests with uninversion */
public class TestFieldCacheSortRandom extends LuceneTestCase {

14
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java → solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheVsDocValues.java
View File

@ -14,9 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
package org.apache.solr.uninverting;
import java.util.ArrayList;
import java.util.Collections;
@ -31,26 +29,28 @@ import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.index.SlowCompositeReaderWrapper;
import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;
public class TestFieldCacheVsDocValues extends LuceneTestCase {

9
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java → solr/core/src/test/org/apache/solr/uninverting/TestFieldCacheWithThreads.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.io.IOException;
import java.util.ArrayList;
@ -23,17 +23,18 @@ import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SortedDocValues;
@ -183,7 +184,7 @@ public class TestFieldCacheWithThreads extends LuceneTestCase {
final LeafReader sr = getOnlyLeafReader(r);
final long END_TIME = System.currentTimeMillis() + (TEST_NIGHTLY ? 30 : 1);
final long END_TIME = System.nanoTime() + TimeUnit.NANOSECONDS.convert((TEST_NIGHTLY ? 30 : 1), TimeUnit.SECONDS);
final int NUM_THREADS = TestUtil.nextInt(random(), 1, 10);
Thread[] threads = new Thread[NUM_THREADS];
@ -201,7 +202,7 @@ public class TestFieldCacheWithThreads extends LuceneTestCase {
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
while(System.currentTimeMillis() < END_TIME) {
while(System.nanoTime() < END_TIME) {
final SortedDocValues source;
source = stringDVDirect;

11
lucene/misc/src/test/org/apache/lucene/uninverting/TestLegacyFieldCache.java → solr/core/src/test/org/apache/solr/uninverting/TestLegacyFieldCache.java
View File

@ -14,12 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@ -27,9 +26,9 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.LegacyDoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LegacyDoubleField;
import org.apache.lucene.document.LegacyFloatField;
import org.apache.lucene.document.LegacyIntField;
import org.apache.lucene.document.LegacyLongField;
@ -37,14 +36,13 @@ import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.Directory;
@ -54,6 +52,7 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LegacyNumericUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.junit.AfterClass;
import org.junit.BeforeClass;

4
lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms32.java → solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms32.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.util.HashMap;
import java.util.Map;
@ -33,9 +33,9 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

4
lucene/misc/src/test/org/apache/lucene/uninverting/TestNumericTerms64.java → solr/core/src/test/org/apache/solr/uninverting/TestNumericTerms64.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.util.HashMap;
import java.util.Map;
@ -33,9 +33,9 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.uninverting.UninvertingReader.Type;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

18
lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java → solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
View File

@ -14,42 +14,42 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.uninverting;
package org.apache.solr.uninverting;
import java.io.IOException;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import java.util.Collections;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LegacyIntField;
import org.apache.lucene.document.LegacyLongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LegacyNumericUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.apache.solr.uninverting.UninvertingReader.Type;
public class TestUninvertingReader extends LuceneTestCase {

13
solr/core/src/test/org/apache/solr/update/DocumentBuilderTest.java
View File

@ -22,25 +22,24 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.core.SolrCore;
import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.ResultContext;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.DocList;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.CopyField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher;
import org.junit.BeforeClass;
import org.junit.Test;

2
solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
View File

@ -503,7 +503,7 @@ public class FacetStream extends TupleStream implements Expressible {
t.put(identifier, d);
++m;
} else {
long l = (long)bucket.get("count");
long l = ((Number)bucket.get("count")).longValue();
t.put("count(*)", l);
}
}

48
solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
View File

@ -73,14 +73,19 @@ public class StreamingTest extends SolrCloudTestCase {
.withFunctionName("parallel", ParallelStream.class);
private static String zkHost;
private static int numShards;
private static int numWorkers;
@BeforeClass
public static void configureCluster() throws Exception {
configureCluster(2)
numShards = random().nextInt(2) + 1; //1 - 3
numWorkers = numShards > 2 ? random().nextInt(numShards - 1) + 1 : numShards;
configureCluster(numShards)
.addConfig("conf", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("streaming").resolve("conf"))
.configure();
CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 1).process(cluster.getSolrClient());
CollectionAdminRequest.createCollection(COLLECTION, "conf", numShards, 1).process(cluster.getSolrClient());
AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(), false, true, TIMEOUT);
zkHost = cluster.getZkServer().getZkAddress();
@ -147,12 +152,11 @@ public class StreamingTest extends SolrCloudTestCase {
SolrParams sParamsA = StreamingTest.mapParams("q", "*:*", "fl", "id,a_s,a_i,a_f", "sort", "a_s asc,a_f asc", "partitionKeys", "none");
CloudSolrStream stream = new CloudSolrStream(zkHost, COLLECTION, sParamsA);
ParallelStream pstream = new ParallelStream(zkHost, COLLECTION, stream, 2, new FieldComparator("a_s",ComparatorOrder.ASCENDING));
ParallelStream pstream = parallelStream(stream, new FieldComparator("a_s", ComparatorOrder.ASCENDING));
attachStreamFactory(pstream);
List<Tuple> tuples = getTuples(pstream);
assert(tuples.size() == 20); // Each tuple will be double counted.
assert(tuples.size() == (10 * numWorkers)); // Each tuple will be double counted.
}
@ -174,7 +178,7 @@ public class StreamingTest extends SolrCloudTestCase {
SolrParams sParams = mapParams("q", "*:*", "fl", "id,a_s,a_i,a_f", "sort", "a_f asc,a_i asc", "partitionKeys", "a_f");
CloudSolrStream stream = new CloudSolrStream(zkHost, COLLECTION, sParams);
UniqueStream ustream = new UniqueStream(stream, new FieldEqualitor("a_f"));
ParallelStream pstream = new ParallelStream(zkHost, COLLECTION, ustream, 2, new FieldComparator("a_f",ComparatorOrder.ASCENDING));
ParallelStream pstream = parallelStream(ustream, new FieldComparator("a_f", ComparatorOrder.ASCENDING));
attachStreamFactory(pstream);
List<Tuple> tuples = getTuples(pstream);
assert(tuples.size() == 5);
@ -183,7 +187,7 @@ public class StreamingTest extends SolrCloudTestCase {
//Test the eofTuples
Map<String,Tuple> eofTuples = pstream.getEofTuples();
assert(eofTuples.size() == 2); //There should be an EOF tuple for each worker.
assert(eofTuples.size() == numWorkers); //There should be an EOF tuple for each worker.
}
@ -253,7 +257,7 @@ public class StreamingTest extends SolrCloudTestCase {
SolrParams sParams = mapParams("q", "*:*", "fl", "id,a_s,a_i", "sort", "a_i asc", "partitionKeys", "a_i");
CloudSolrStream stream = new CloudSolrStream(zkHost, COLLECTION, sParams);
RankStream rstream = new RankStream(stream, 11, new FieldComparator("a_i",ComparatorOrder.DESCENDING));
ParallelStream pstream = new ParallelStream(zkHost, COLLECTION, rstream, 2, new FieldComparator("a_i",ComparatorOrder.DESCENDING));
ParallelStream pstream = parallelStream(rstream, new FieldComparator("a_i", ComparatorOrder.DESCENDING));
attachStreamFactory(pstream);
List<Tuple> tuples = getTuples(pstream);
@ -405,9 +409,7 @@ public class StreamingTest extends SolrCloudTestCase {
ReducerStream rstream = new ReducerStream(stream,
new FieldEqualitor("a_s"),
new GroupOperation(new FieldComparator("a_f", ComparatorOrder.DESCENDING), 5));
ParallelStream pstream = new ParallelStream(zkHost, COLLECTION, rstream, 2, new FieldComparator("a_s",ComparatorOrder.ASCENDING));
ParallelStream pstream = parallelStream(rstream, new FieldComparator("a_s", ComparatorOrder.ASCENDING));
attachStreamFactory(pstream);
List<Tuple> tuples = getTuples(pstream);
@ -433,9 +435,7 @@ public class StreamingTest extends SolrCloudTestCase {
rstream = new ReducerStream(stream,
new FieldEqualitor("a_s"),
new GroupOperation(new FieldComparator("a_f", ComparatorOrder.ASCENDING), 3));
pstream = new ParallelStream(zkHost, COLLECTION, rstream, 2, new FieldComparator("a_s",ComparatorOrder.DESCENDING));
pstream = parallelStream(rstream, new FieldComparator("a_s", ComparatorOrder.DESCENDING));
attachStreamFactory(pstream);
tuples = getTuples(pstream);
@ -1401,7 +1401,7 @@ public class StreamingTest extends SolrCloudTestCase {
new CountMetric()};
RollupStream rollupStream = new RollupStream(stream, buckets, metrics);
ParallelStream parallelStream = new ParallelStream(zkHost, COLLECTION, rollupStream, 2, new FieldComparator("a_s", ComparatorOrder.ASCENDING));
ParallelStream parallelStream = parallelStream(rollupStream, new FieldComparator("a_s", ComparatorOrder.ASCENDING));
attachStreamFactory(parallelStream);
List<Tuple> tuples = getTuples(parallelStream);
@ -1501,9 +1501,7 @@ public class StreamingTest extends SolrCloudTestCase {
ReducerStream rstream = new ReducerStream(stream,
new FieldEqualitor("a_s"),
new GroupOperation(new FieldComparator("a_s", ComparatorOrder.ASCENDING), 2));
ParallelStream pstream = new ParallelStream(zkHost, COLLECTION, rstream, 2, new FieldComparator("a_s", ComparatorOrder.ASCENDING));
ParallelStream pstream = parallelStream(rstream, new FieldComparator("a_s", ComparatorOrder.ASCENDING));
attachStreamFactory(pstream);
List<Tuple> tuples = getTuples(pstream);
assert(tuples.size() == 0);
@ -1636,7 +1634,7 @@ public class StreamingTest extends SolrCloudTestCase {
CloudSolrStream streamB = new CloudSolrStream(zkHost, COLLECTION, sParamsB);
MergeStream mstream = new MergeStream(streamA, streamB, new FieldComparator("a_i",ComparatorOrder.ASCENDING));
ParallelStream pstream = new ParallelStream(zkHost, COLLECTION, mstream, 2, new FieldComparator("a_i",ComparatorOrder.ASCENDING));
ParallelStream pstream = parallelStream(mstream, new FieldComparator("a_i", ComparatorOrder.ASCENDING));
attachStreamFactory(pstream);
List<Tuple> tuples = getTuples(pstream);
@ -1651,7 +1649,7 @@ public class StreamingTest extends SolrCloudTestCase {
streamB = new CloudSolrStream(zkHost, COLLECTION, sParamsB);
mstream = new MergeStream(streamA, streamB, new FieldComparator("a_i",ComparatorOrder.DESCENDING));
pstream = new ParallelStream(zkHost, COLLECTION, mstream, 2, new FieldComparator("a_i",ComparatorOrder.DESCENDING));
pstream = parallelStream(mstream, new FieldComparator("a_i", ComparatorOrder.DESCENDING));
attachStreamFactory(pstream);
tuples = getTuples(pstream);
@ -1684,14 +1682,13 @@ public class StreamingTest extends SolrCloudTestCase {
CloudSolrStream streamB = new CloudSolrStream(zkHost, COLLECTION, sParamsB);
MergeStream mstream = new MergeStream(streamA, streamB, new FieldComparator("a_i",ComparatorOrder.ASCENDING));
ParallelStream pstream = new ParallelStream(zkHost, COLLECTION, mstream, 2, new FieldComparator("a_i",ComparatorOrder.ASCENDING));
ParallelStream pstream = parallelStream(mstream, new FieldComparator("a_i", ComparatorOrder.ASCENDING));
attachStreamFactory(pstream);
List<Tuple> tuples = getTuples(pstream);
assert(tuples.size() == 9);
Map<String, Tuple> eofTuples = pstream.getEofTuples();
assert(eofTuples.size() == 2); // There should be an EOF Tuple for each worker.
assert(eofTuples.size() == numWorkers); // There should be an EOF Tuple for each worker.
}
@ -1834,5 +1831,10 @@ public class StreamingTest extends SolrCloudTestCase {
return params;
}
private ParallelStream parallelStream(TupleStream stream, FieldComparator comparator) throws IOException {
ParallelStream pstream = new ParallelStream(zkHost, COLLECTION, stream, numWorkers, comparator);
return pstream;
}
}