Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'sequence_numbers' of github.com:mikemccand/lucene-solr into sequence_numbers

This commit is contained in:
Mike McCandless 2016-06-01 18:51:10 -04:00
parent 293314bf6d 671de29635
commit d44f87d8b5
33 changed files with 1214 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
dev-tools/scripts/checkJavaDocs.py
View File

@ -78,7 +78,11 @@ def cleanHTML(s):
reH3 = re.compile('^<h3>(.*?)</h3>', re.IGNORECASE | re.MULTILINE)
reH4 = re.compile('^<h4>(.*?)</h4>', re.IGNORECASE | re.MULTILINE)
reDetailsDiv = re.compile('<div class="details">')
reEndOfClassData = re.compile('<!--.*END OF CLASS DATA.*-->')
reBlockList = re.compile('<ul class="blockList(?:Last)?">')
reCloseUl = re.compile('</ul>')
def checkClassDetails(fullPath):
"""
Checks for invalid HTML in the full javadocs under each field/method.
@ -86,60 +90,54 @@ def checkClassDetails(fullPath):
# TODO: only works with java7 generated javadocs now!
with open(fullPath, encoding='UTF-8') as f:
desc = None
desc = []
cat = None
item = None
errors = []
inDetailsDiv = False
blockListDepth = 0
for line in f.readlines():
# Skip content up until <div class="details">
if not inDetailsDiv:
if reDetailsDiv.match(line) is not None:
inDetailsDiv = True
continue
m = reH3.search(line)
if m is not None:
if desc is not None:
desc = ''.join(desc)
if True or cat == 'Constructor Detail':
idx = desc.find('</div>')
if idx == -1:
# Ctor missing javadocs ... checkClassSummaries catches it
desc = None
continue
desc = desc[:idx+6]
else:
# Have to fake <ul> context because we pulled a fragment out "across" two <ul>s:
desc = '<ul>%s</ul>' % ''.join(desc)
#print(' VERIFY %s: %s: %s' % (cat, item, desc))
# Stop looking at content at closing details </div>, which is just before <!-- === END OF CLASS DATA === -->
if reEndOfClassData.match(line) is not None:
if len(desc) != 0:
try:
verifyHTML(desc)
verifyHTML(''.join(desc))
except RuntimeError as re:
#print(' FAILED: %s' % re)
errors.append((cat, item, str(re)))
desc = None
cat = m.group(1)
continue
break
m = reH4.search(line)
if m is not None:
if desc is not None:
# Have to fake <ul> context because we pulled a fragment out "across" two <ul>s:
if cat == 'Element Detail':
desc = ''.join(desc)
idx = desc.find('</dl>')
if idx != -1:
desc = desc[:idx+5]
else:
desc = '<ul>%s</ul>' % ''.join(desc)
#print(' VERIFY %s: %s: %s' % (cat, item, desc))
# <ul class="blockList(Last)"> is the boundary between items
if reBlockList.match(line) is not None:
blockListDepth += 1
if len(desc) != 0:
try:
verifyHTML(desc)
verifyHTML(''.join(desc))
except RuntimeError as re:
#print(' FAILED: %s' % re)
errors.append((cat, item, str(re)))
item = m.group(1)
desc = []
continue
del desc[:]
if desc is not None:
if blockListDepth == 3:
desc.append(line)
if reCloseUl.match(line) is not None:
blockListDepth -= 1
else:
m = reH3.search(line)
if m is not None:
cat = m.group(1)
else:
m = reH4.search(line)
if m is not None:
item = m.group(1)
if len(errors) != 0:
print()
print(fullPath)

4
lucene/CHANGES.txt
View File

@ -137,6 +137,10 @@ Bug Fixes
* LUCENE-7293: Don't try to highlight GeoPoint queries (Britta Weber,
Nick Knize, Mike McCandless, Uwe Schindler)
* LUCENE-7301: Multiple doc values updates to the same document within
one update batch could be applied in the wrong order resulting in
the wrong updated value (Ishan Chattopadhyaya, hossman, Mike McCandless)
Documentation
* LUCENE-7223: Improve XXXPoint javadocs to make it clear that you

18
lucene/core/src/java/org/apache/lucene/geo/Polygon2D.java
View File

@ -17,9 +17,11 @@
package org.apache.lucene.geo;
import java.util.Arrays;
import java.util.Comparator;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.index.PointValues.Relation;
import org.apache.lucene.util.ArrayUtil;
/**
* 2D polygon implementation represented as a balanced interval tree of edges.
@ -209,28 +211,30 @@ public final class Polygon2D {
private static Polygon2D createTree(Polygon2D components[], int low, int high, boolean splitX) {
if (low > high) {
return null;
} else if (low < high) {
// TODO: do one sort instead! there are better algorithms!
}
final int mid = (low + high) >>> 1;
if (low < high) {
Comparator<Polygon2D> comparator;
if (splitX) {
Arrays.sort(components, low, high+1, (left, right) -> {
comparator = (left, right) -> {
int ret = Double.compare(left.minLon, right.minLon);
if (ret == 0) {
ret = Double.compare(left.maxX, right.maxX);
}
return ret;
});
};
} else {
Arrays.sort(components, low, high+1, (left, right) -> {
comparator = (left, right) -> {
int ret = Double.compare(left.minLat, right.minLat);
if (ret == 0) {
ret = Double.compare(left.maxY, right.maxY);
}
return ret;
});
};
}
ArrayUtil.select(components, low, high + 1, mid, comparator);
}
// add midpoint
int mid = (low + high) >>> 1;
Polygon2D newNode = components[mid];
newNode.splitX = splitX;
// add children

5
lucene/core/src/java/org/apache/lucene/index/BufferedUpdates.java
View File

@ -158,9 +158,12 @@ class BufferedUpdates {
private final static boolean VERBOSE_DELETES = false;
long gen;
final String segmentName;
public BufferedUpdates() {
public BufferedUpdates(String segmentName) {
this.bytesUsed = new AtomicLong();
this.segmentName = segmentName;
}
@Override

30
lucene/core/src/java/org/apache/lucene/index/BufferedUpdatesStream.java
View File

@ -228,14 +228,19 @@ class BufferedUpdatesStream implements Accountable {
assert pool.infoIsLive(info);
int delCount = 0;
final DocValuesFieldUpdates.Container dvUpdates = new DocValuesFieldUpdates.Container();
if (coalescedUpdates != null) {
delCount += applyQueryDeletes(coalescedUpdates.queriesIterable(), segState);
applyDocValuesUpdates(coalescedUpdates.numericDVUpdates, segState, dvUpdates);
applyDocValuesUpdates(coalescedUpdates.binaryDVUpdates, segState, dvUpdates);
}
// first apply segment-private deletes/updates
delCount += applyQueryDeletes(packet.queriesIterable(), segState);
applyDocValuesUpdates(Arrays.asList(packet.numericDVUpdates), segState, dvUpdates);
applyDocValuesUpdates(Arrays.asList(packet.binaryDVUpdates), segState, dvUpdates);
// ... then coalesced deletes/updates, so that if there is an update that appears in both, the coalesced updates (carried from
// updates ahead of the segment-privates ones) win:
if (coalescedUpdates != null) {
delCount += applyQueryDeletes(coalescedUpdates.queriesIterable(), segState);
applyDocValuesUpdatesList(coalescedUpdates.numericDVUpdates, segState, dvUpdates);
applyDocValuesUpdatesList(coalescedUpdates.binaryDVUpdates, segState, dvUpdates);
}
if (dvUpdates.any()) {
segState.rld.writeFieldUpdates(info.info.dir, dvUpdates);
}
@ -261,8 +266,8 @@ class BufferedUpdatesStream implements Accountable {
int delCount = 0;
delCount += applyQueryDeletes(coalescedUpdates.queriesIterable(), segState);
DocValuesFieldUpdates.Container dvUpdates = new DocValuesFieldUpdates.Container();
applyDocValuesUpdates(coalescedUpdates.numericDVUpdates, segState, dvUpdates);
applyDocValuesUpdates(coalescedUpdates.binaryDVUpdates, segState, dvUpdates);
applyDocValuesUpdatesList(coalescedUpdates.numericDVUpdates, segState, dvUpdates);
applyDocValuesUpdatesList(coalescedUpdates.binaryDVUpdates, segState, dvUpdates);
if (dvUpdates.any()) {
segState.rld.writeFieldUpdates(info.info.dir, dvUpdates);
}
@ -599,8 +604,17 @@ class BufferedUpdatesStream implements Accountable {
return delTermVisitedCount;
}
private synchronized void applyDocValuesUpdatesList(List<List<DocValuesUpdate>> updates,
SegmentState segState, DocValuesFieldUpdates.Container dvUpdatesContainer) throws IOException {
// we walk backwards through the segments, appending deletion packets to the coalesced updates, so we must apply the packets in reverse
// so that newer packets override older ones:
for(int idx=updates.size()-1;idx>=0;idx--) {
applyDocValuesUpdates(updates.get(idx), segState, dvUpdatesContainer);
}
}
// DocValues updates
private synchronized void applyDocValuesUpdates(Iterable<? extends DocValuesUpdate> updates,
private synchronized void applyDocValuesUpdates(List<DocValuesUpdate> updates,
SegmentState segState, DocValuesFieldUpdates.Container dvUpdatesContainer) throws IOException {
Fields fields = segState.reader.fields();

14
lucene/core/src/java/org/apache/lucene/index/CoalescedUpdates.java
View File

@ -32,8 +32,8 @@ import org.apache.lucene.util.BytesRef;
class CoalescedUpdates {
final Map<Query,Integer> queries = new HashMap<>();
final List<PrefixCodedTerms> terms = new ArrayList<>();
final List<NumericDocValuesUpdate> numericDVUpdates = new ArrayList<>();
final List<BinaryDocValuesUpdate> binaryDVUpdates = new ArrayList<>();
final List<List<DocValuesUpdate>> numericDVUpdates = new ArrayList<>();
final List<List<DocValuesUpdate>> binaryDVUpdates = new ArrayList<>();
long totalTermCount;
@Override
@ -53,17 +53,21 @@ class CoalescedUpdates {
final Query query = in.queries[queryIdx];
queries.put(query, BufferedUpdates.MAX_INT);
}
List<DocValuesUpdate> numericPacket = new ArrayList<>();
numericDVUpdates.add(numericPacket);
for (NumericDocValuesUpdate nu : in.numericDVUpdates) {
NumericDocValuesUpdate clone = new NumericDocValuesUpdate(nu.term, nu.field, (Long) nu.value);
clone.docIDUpto = Integer.MAX_VALUE;
numericDVUpdates.add(clone);
numericPacket.add(clone);
}
List<DocValuesUpdate> binaryPacket = new ArrayList<>();
binaryDVUpdates.add(binaryPacket);
for (BinaryDocValuesUpdate bu : in.binaryDVUpdates) {
BinaryDocValuesUpdate clone = new BinaryDocValuesUpdate(bu.term, bu.field, (BytesRef) bu.value);
clone.docIDUpto = Integer.MAX_VALUE;
binaryDVUpdates.add(clone);
binaryPacket.add(clone);
}
}

2
lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
View File

@ -68,7 +68,7 @@ abstract class DocValuesUpdate {
@Override
public String toString() {
return "term=" + term + ",field=" + field + ",value=" + value;
return "term=" + term + ",field=" + field + ",value=" + value + ",docIDUpto=" + docIDUpto;
}
/** An in-place update to a binary DocValues field */

3
lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
View File

@ -626,7 +626,7 @@ final class DocumentsWriter implements Closeable, Accountable {
/* Cutover to a new delete queue. This must be synced on the flush control
* otherwise a new DWPT could sneak into the loop with an already flushing
* delete queue */
seqNo = flushControl.markForFullFlush(); // swaps the delQueue synced on FlushControl
seqNo = flushControl.markForFullFlush(); // swaps this.deleteQueue synced on FlushControl
assert setFlushingDeleteQueue(flushingDeleteQueue);
}
assert currentFullFlushDelQueue != null;
@ -676,7 +676,6 @@ final class DocumentsWriter implements Closeable, Accountable {
} finally {
pendingChangesInCurrentFullFlush = false;
}
}
public LiveIndexWriterConfig getIndexWriterConfig() {

69
lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
View File

@ -69,12 +69,15 @@ import org.apache.lucene.util.BytesRef;
*/
final class DocumentsWriterDeleteQueue implements Accountable {
// the current end (latest delete operation) in the delete queue:
private volatile Node<?> tail;
@SuppressWarnings("rawtypes")
private static final AtomicReferenceFieldUpdater<DocumentsWriterDeleteQueue,Node> tailUpdater = AtomicReferenceFieldUpdater
.newUpdater(DocumentsWriterDeleteQueue.class, Node.class, "tail");
/** Used to record deletes against all prior (already written to disk) segments. Whenever any segment flushes, we bundle up this set of
* deletes and insert into the buffered updates stream before the newly flushed segment(s). */
private final DeleteSlice globalSlice;
private final BufferedUpdates globalBufferedUpdates;
@ -85,6 +88,9 @@ final class DocumentsWriterDeleteQueue implements Accountable {
/** Generates the sequence number that IW returns to callers changing the index, showing the effective serialization of all operations. */
private final AtomicLong nextSeqNo;
// for asserts
long maxSeqNo = Long.MAX_VALUE;
DocumentsWriterDeleteQueue() {
// seqNo must start at 1 because some APIs negate this to also return a boolean
@ -92,7 +98,7 @@ final class DocumentsWriterDeleteQueue implements Accountable {
}
DocumentsWriterDeleteQueue(long generation, long startSeqNo) {
this(new BufferedUpdates(), generation, startSeqNo);
this(new BufferedUpdates("global"), generation, startSeqNo);
}
DocumentsWriterDeleteQueue(BufferedUpdates globalBufferedUpdates, long generation, long startSeqNo) {
@ -130,7 +136,6 @@ final class DocumentsWriterDeleteQueue implements Accountable {
*/
long add(Term term, DeleteSlice slice) {
final TermNode termNode = new TermNode(term);
// System.out.println(Thread.currentThread().getName() + ": push " + termNode + " this=" + this);
long seqNo = add(termNode);
/*
* this is an update request where the term is the updated documents
@ -150,31 +155,10 @@ final class DocumentsWriterDeleteQueue implements Accountable {
return seqNo;
}
long add(Node<?> newNode) {
/*
* this non-blocking / 'wait-free' linked list add was inspired by Apache
* Harmony's ConcurrentLinkedQueue Implementation.
*/
while (true) {
final Node<?> currentTail = tail;
final Node<?> tailNext = currentTail.next;
if (tail == currentTail && tailNext == null) {
/*
* we are in quiescent state and can try to insert the newNode to the
* current tail if we fail to insert we just retry the operation since
* somebody else has already added its newNode
*/
if (currentTail.casNext(null, newNode)) {
/*
* now that we are done we need to advance the tail
*/
long seqNo = getNextSequenceNumber();
boolean result = tailUpdater.compareAndSet(this, currentTail, newNode);
assert result;
return seqNo;
}
}
}
synchronized long add(Node<?> newNode) {
tail.next = newNode;
this.tail = newNode;
return getNextSequenceNumber();
}
boolean anyChanges() {
@ -185,8 +169,7 @@ final class DocumentsWriterDeleteQueue implements Accountable {
* and if the global slice is up-to-date
* and if globalBufferedUpdates has changes
*/
return globalBufferedUpdates.any() || !globalSlice.isEmpty() || globalSlice.sliceTail != tail
|| tail.next != null;
return globalBufferedUpdates.any() || !globalSlice.isEmpty() || globalSlice.sliceTail != tail || tail.next != null;
} finally {
globalBufferLock.unlock();
}
@ -201,8 +184,7 @@ final class DocumentsWriterDeleteQueue implements Accountable {
* tail the next time we can get the lock!
*/
try {
if (updateSlice(globalSlice)) {
// System.out.println(Thread.currentThread() + ": apply globalSlice");
if (updateSliceNoSeqNo(globalSlice)) {
globalSlice.apply(globalBufferedUpdates, BufferedUpdates.MAX_INT);
}
} finally {
@ -231,7 +213,6 @@ final class DocumentsWriterDeleteQueue implements Accountable {
globalSlice.apply(globalBufferedUpdates, BufferedUpdates.MAX_INT);
}
// System.out.println(Thread.currentThread().getName() + ": now freeze global buffer " + globalBufferedDeletes);
final FrozenBufferedUpdates packet = new FrozenBufferedUpdates(globalBufferedUpdates, false);
globalBufferedUpdates.clear();
return packet;
@ -244,8 +225,21 @@ final class DocumentsWriterDeleteQueue implements Accountable {
return new DeleteSlice(tail);
}
boolean updateSlice(DeleteSlice slice) {
if (slice.sliceTail != tail) { // If we are the same just
/** Negative result means there were new deletes since we last applied */
synchronized long updateSlice(DeleteSlice slice) {
long seqNo = getNextSequenceNumber();
if (slice.sliceTail != tail) {
// new deletes arrived since we last checked
slice.sliceTail = tail;
seqNo = -seqNo;
}
return seqNo;
}
/** Just like updateSlice, but does not assign a sequence number */
boolean updateSliceNoSeqNo(DeleteSlice slice) {
if (slice.sliceTail != tail) {
// new deletes arrived since we last checked
slice.sliceTail = tail;
return true;
}
@ -283,7 +277,6 @@ final class DocumentsWriterDeleteQueue implements Accountable {
current = current.next;
assert current != null : "slice property violated between the head on the tail must not be a null node";
current.apply(del, docIDUpto);
// System.out.println(Thread.currentThread().getName() + ": pull " + current + " docIDUpto=" + docIDUpto);
} while (current != sliceTail);
reset();
}
@ -462,13 +455,17 @@ final class DocumentsWriterDeleteQueue implements Accountable {
}
public long getNextSequenceNumber() {
return nextSeqNo.getAndIncrement();
long seqNo = nextSeqNo.getAndIncrement();
assert seqNo < maxSeqNo: "seqNo=" + seqNo + " vs maxSeqNo=" + maxSeqNo;
return seqNo;
}
public long getLastSequenceNumber() {
return nextSeqNo.get()-1;
}
/** Inserts a gap in the sequence numbers. This is used by IW during flush or commit to ensure any in-flight threads get sequence numbers
* inside the gap */
public void skipSequenceNumbers(long jump) {
nextSeqNo.addAndGet(jump);
}

7
lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
View File

@ -190,7 +190,7 @@ final class DocumentsWriterFlushControl implements Accountable {
flushingDWPT = null;
}
} else {
flushingDWPT = tryCheckoutForFlush(perThread);
flushingDWPT = tryCheckoutForFlush(perThread);
}
return flushingDWPT;
} finally {
@ -452,8 +452,7 @@ final class DocumentsWriterFlushControl implements Accountable {
.currentThread(), documentsWriter);
boolean success = false;
try {
if (perThread.isInitialized()
&& perThread.dwpt.deleteQueue != documentsWriter.deleteQueue) {
if (perThread.isInitialized() && perThread.dwpt.deleteQueue != documentsWriter.deleteQueue) {
// There is a flush-all in process and this DWPT is
// now stale -- enroll it for flush and try for
// another DWPT:
@ -479,11 +478,11 @@ final class DocumentsWriterFlushControl implements Accountable {
flushingQueue = documentsWriter.deleteQueue;
// Set a new delete queue - all subsequent DWPT will use this queue until
// we do another full flush
//System.out.println("DWFC: fullFLush old seqNo=" + documentsWriter.deleteQueue.seqNo.get() + " activeThreadCount=" + perThreadPool.getActiveThreadStateCount());
// Insert a gap in seqNo of current active thread count, in the worst case each of those threads now have one operation in flight. It's fine
// if we have some sequence numbers that were never assigned:
seqNo = documentsWriter.deleteQueue.getLastSequenceNumber() + perThreadPool.getActiveThreadStateCount() + 2;
flushingQueue.maxSeqNo = seqNo+1;
DocumentsWriterDeleteQueue newQueue = new DocumentsWriterDeleteQueue(flushingQueue.generation+1, seqNo+1);

22
lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
View File

@ -171,7 +171,7 @@ class DocumentsWriterPerThread {
this.pendingNumDocs = pendingNumDocs;
bytesUsed = Counter.newCounter();
byteBlockAllocator = new DirectTrackingAllocator(bytesUsed);
pendingUpdates = new BufferedUpdates();
pendingUpdates = new BufferedUpdates(segmentName);
intBlockAllocator = new IntBlockAllocator(bytesUsed);
this.deleteQueue = deleteQueue;
assert numDocsInRAM == 0 : "num docs " + numDocsInRAM;
@ -278,7 +278,8 @@ class DocumentsWriterPerThread {
numDocsInRAM++;
}
}
finishDocument(null);
numDocsInRAM++;
}
allDocsIndexed = true;
@ -292,7 +293,13 @@ class DocumentsWriterPerThread {
deleteSlice.apply(pendingUpdates, numDocsInRAM-docCount);
return seqNo;
} else {
seqNo = deleteQueue.getNextSequenceNumber();
seqNo = deleteQueue.updateSlice(deleteSlice);
if (seqNo < 0) {
seqNo = -seqNo;
deleteSlice.apply(pendingUpdates, numDocsInRAM-docCount);
} else {
deleteSlice.reset();
}
}
return seqNo;
@ -327,8 +334,13 @@ class DocumentsWriterPerThread {
seqNo = deleteQueue.add(delTerm, deleteSlice);
assert deleteSlice.isTailItem(delTerm) : "expected the delete term as the tail item";
} else {
applySlice &= deleteQueue.updateSlice(deleteSlice);
seqNo = deleteQueue.getNextSequenceNumber();
seqNo = deleteQueue.updateSlice(deleteSlice);
if (seqNo < 0) {
seqNo = -seqNo;
} else {
applySlice = false;
}
}
if (applySlice) {

1
lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
View File

@ -226,6 +226,7 @@ final class DocumentsWriterPerThreadPool {
return threadStates.get(ord);
}
// TODO: merge this with getActiveThreadStateCount: they are the same!
synchronized int getMaxThreadStates() {
return threadStates.size();
}

71
lucene/core/src/java/org/apache/lucene/util/ArrayUtil.java
View File

@ -453,4 +453,75 @@ public final class ArrayUtil {
timSort(a, 0, a.length);
}
/** Reorganize {@code arr[from:to[} so that the element at offset k is at the
* same position as if {@code arr[from:to[} was sorted, and all elements on
* its left are less than or equal to it, and all elements on its right are
* greater than or equal to it.
* This runs in linear time on average and in {@code n log(n)} time in the
* worst case.*/
public static <T> void select(T[] arr, int from, int to, int k, Comparator<T> comparator) {
if (k < from) {
throw new IllegalArgumentException("k must be >= from");
}
if (k >= to) {
throw new IllegalArgumentException("k must be < to");
}
final int maxDepth = 2 * MathUtil.log(to - from, 2);
quickSelect(arr, from, to, k, comparator, maxDepth);
}
private static <T> void quickSelect(T[] arr, int from, int to, int k, Comparator<T> comparator, int maxDepth) {
assert from <= k;
assert k < to;
if (to - from == 1) {
return;
}
if (--maxDepth < 0) {
Arrays.sort(arr, from, to, comparator);
return;
}
final int mid = (from + to) >>> 1;
// heuristic: we use the median of the values at from, to-1 and mid as a pivot
if (comparator.compare(arr[from], arr[to - 1]) > 0) {
swap(arr, from, to - 1);
}
if (comparator.compare(arr[to - 1], arr[mid]) > 0) {
swap(arr, to - 1, mid);
if (comparator.compare(arr[from], arr[to - 1]) > 0) {
swap(arr, from, to - 1);
}
}
T pivot = arr[to - 1];
int left = from + 1;
int right = to - 2;
for (;;) {
while (comparator.compare(pivot, arr[left]) > 0) {
++left;
}
while (left < right && comparator.compare(pivot, arr[right]) <= 0) {
--right;
}
if (left < right) {
swap(arr, left, right);
--right;
} else {
break;
}
}
swap(arr, left, to - 1);
if (left == k) {
return;
} else if (left < k) {
quickSelect(arr, left + 1, to, k, comparator, maxDepth);
} else {
quickSelect(arr, from, left, k, comparator, maxDepth);
}
}
}

6
lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java
View File

@ -43,8 +43,8 @@ public class TestDocumentsWriterDeleteQueue extends LuceneTestCase {
}
DeleteSlice slice1 = queue.newSlice();
DeleteSlice slice2 = queue.newSlice();
BufferedUpdates bd1 = new BufferedUpdates();
BufferedUpdates bd2 = new BufferedUpdates();
BufferedUpdates bd1 = new BufferedUpdates("bd1");
BufferedUpdates bd2 = new BufferedUpdates("bd2");
int last1 = 0;
int last2 = 0;
Set<Term> uniqueValues = new HashSet<>();
@ -225,7 +225,7 @@ public class TestDocumentsWriterDeleteQueue extends LuceneTestCase {
this.index = index;
this.ids = ids;
this.slice = queue.newSlice();
deletes = new BufferedUpdates();
deletes = new BufferedUpdates("deletes");
this.latch = latch;
}

1
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
View File

@ -93,7 +93,6 @@ public class TestIndexWriterConfig extends LuceneTestCase {
getters.add("getIndexingChain");
getters.add("getMergedSegmentWarmer");
getters.add("getMergePolicy");
getters.add("getMaxThreadStates");
getters.add("getReaderPooling");
getters.add("getIndexerThreadPool");
getters.add("getFlushPolicy");

236
lucene/core/src/test/org/apache/lucene/index/TestIndexingSequenceNumbers.java
View File

@ -17,6 +17,20 @@
package org.apache.lucene.index;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@ -25,17 +39,6 @@ import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
public class TestIndexingSequenceNumbers extends LuceneTestCase {
public void testBasic() throws Exception {
@ -91,7 +94,13 @@ public class TestIndexingSequenceNumbers extends LuceneTestCase {
doc.add(new StringField("id", "id", Field.Store.NO));
startingGun.await();
for(int j=0;j<100;j++) {
seqNos[threadID] = w.updateDocument(id, doc);
if (random().nextBoolean()) {
seqNos[threadID] = w.updateDocument(id, doc);
} else {
List<Document> docs = new ArrayList<>();
docs.add(doc);
seqNos[threadID] = w.updateDocuments(id, docs);
}
}
} catch (Exception e) {
throw new RuntimeException(e);
@ -147,7 +156,7 @@ public class TestIndexingSequenceNumbers extends LuceneTestCase {
// Cannot use RIW since it randomly commits:
final IndexWriter w = new IndexWriter(dir, iwc);
final int numThreads = TestUtil.nextInt(random(), 2, 5);
final int numThreads = TestUtil.nextInt(random(), 2, 10);
Thread[] threads = new Thread[numThreads];
//System.out.println("TEST: iter=" + iter + " opCount=" + opCount + " idCount=" + idCount + " threadCount=" + threads.length);
final CountDownLatch startingGun = new CountDownLatch(1);
@ -198,7 +207,7 @@ public class TestIndexingSequenceNumbers extends LuceneTestCase {
} else {
op.seqNo = w.updateDocument(idTerm, doc);
}
op.what = 2;
op.what = 0;
}
ops.add(op);
}
@ -241,7 +250,7 @@ public class TestIndexingSequenceNumbers extends LuceneTestCase {
for(Operation op : threadOps.get(threadID)) {
if (op.seqNo <= commitSeqNo && op.seqNo > seqNos[op.id]) {
seqNos[op.id] = op.seqNo;
if (op.what == 2) {
if (op.what == 0) {
expectedThreadIDs[op.id] = threadID;
} else {
expectedThreadIDs[op.id] = -1;
@ -265,7 +274,7 @@ public class TestIndexingSequenceNumbers extends LuceneTestCase {
Document doc = r.document(hits.scoreDocs[0].doc);
int actualThreadID = doc.getField("thread").numericValue().intValue();
if (expectedThreadIDs[id] != actualThreadID) {
System.out.println("FAIL: id=" + id + " expectedThreadID=" + expectedThreadIDs[id] + " vs actualThreadID=" + actualThreadID);
System.out.println("FAIL: id=" + id + " expectedThreadID=" + expectedThreadIDs[id] + " vs actualThreadID=" + actualThreadID + " commitSeqNo=" + commitSeqNo + " numThreads=" + numThreads);
for(int threadID=0;threadID<threadOps.size();threadID++) {
for(Operation op : threadOps.get(threadID)) {
if (id == op.id) {
@ -276,7 +285,168 @@ public class TestIndexingSequenceNumbers extends LuceneTestCase {
assertEquals("id=" + id, expectedThreadIDs[id], actualThreadID);
}
} else if (hits.totalHits != 0) {
System.out.println("FAIL: id=" + id + " expectedThreadID=" + expectedThreadIDs[id] + " vs totalHits=" + hits.totalHits);
System.out.println("FAIL: id=" + id + " expectedThreadID=" + expectedThreadIDs[id] + " vs totalHits=" + hits.totalHits + " commitSeqNo=" + commitSeqNo + " numThreads=" + numThreads);
for(int threadID=0;threadID<threadOps.size();threadID++) {
for(Operation op : threadOps.get(threadID)) {
if (id == op.id) {
System.out.println(" threadID=" + threadID + " seqNo=" + op.seqNo + " " + (op.what == 2 ? "updated" : "del"));
}
}
}
assertEquals(0, hits.totalHits);
}
}
w.close();
r.close();
}
dir.close();
}
public void testStressConcurrentDocValuesUpdatesCommit() throws Exception {
final int opCount = atLeast(10000);
final int idCount = TestUtil.nextInt(random(), 10, 1000);
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig();
iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
// Cannot use RIW since it randomly commits:
final IndexWriter w = new IndexWriter(dir, iwc);
final int numThreads = TestUtil.nextInt(random(), 2, 10);
Thread[] threads = new Thread[numThreads];
//System.out.println("TEST: iter=" + iter + " opCount=" + opCount + " idCount=" + idCount + " threadCount=" + threads.length);
final CountDownLatch startingGun = new CountDownLatch(1);
List<List<Operation>> threadOps = new ArrayList<>();
Object commitLock = new Object();
final List<Operation> commits = new ArrayList<>();
List<Operation> ops1 = new ArrayList<>();
threadOps.add(ops1);
for(int id=0;id<idCount;id++) {
int threadID = 0;
Operation op = new Operation();
op.threadID = threadID;
op.id = id;
Document doc = new Document();
doc.add(new StoredField("thread", threadID));
doc.add(new NumericDocValuesField("thread", threadID));
doc.add(new StringField("id", "" + id, Field.Store.NO));
op.seqNo = w.addDocument(doc);
ops1.add(op);
}
// multiple threads update the same set of documents, and we randomly commit
for(int i=0;i<threads.length;i++) {
final List<Operation> ops;
if (i == 0) {
ops = threadOps.get(0);
} else {
ops = new ArrayList<>();
threadOps.add(ops);
}
final int threadID = i;
threads[i] = new Thread() {
@Override
public void run() {
try {
startingGun.await();
for(int i=0;i<opCount;i++) {
Operation op = new Operation();
op.threadID = threadID;
if (random().nextInt(500) == 17) {
op.what = 2;
synchronized(commitLock) {
op.seqNo = w.commit();
if (op.seqNo != -1) {
commits.add(op);
}
}
} else {
op.id = random().nextInt(idCount);
Term idTerm = new Term("id", "" + op.id);
op.seqNo = w.updateNumericDocValue(idTerm, "thread", threadID);
op.what = 0;
ops.add(op);
}
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
threads[i].start();
}
startingGun.countDown();
for(Thread thread : threads) {
thread.join();
}
Operation commitOp = new Operation();
commitOp.seqNo = w.commit();
if (commitOp.seqNo != -1) {
commits.add(commitOp);
}
List<IndexCommit> indexCommits = DirectoryReader.listCommits(dir);
assertEquals(commits.size(), indexCommits.size());
int[] expectedThreadIDs = new int[idCount];
long[] seqNos = new long[idCount];
//System.out.println("TEST: " + commits.size() + " commits");
for(int i=0;i<commits.size();i++) {
// this commit point should reflect all operations <= this seqNo
long commitSeqNo = commits.get(i).seqNo;
//System.out.println(" commit " + i + ": seqNo=" + commitSeqNo + " segs=" + indexCommits.get(i));
Arrays.fill(expectedThreadIDs, -1);
Arrays.fill(seqNos, 0);
for(int threadID=0;threadID<threadOps.size();threadID++) {
long lastSeqNo = 0;
for(Operation op : threadOps.get(threadID)) {
if (op.seqNo <= commitSeqNo && op.seqNo > seqNos[op.id]) {
seqNos[op.id] = op.seqNo;
if (op.what == 0) {
expectedThreadIDs[op.id] = threadID;
}
}
assertTrue(op.seqNo > lastSeqNo);
lastSeqNo = op.seqNo;
}
}
DirectoryReader r = DirectoryReader.open(indexCommits.get(i));
IndexSearcher s = new IndexSearcher(r);
NumericDocValues docValues = MultiDocValues.getNumericValues(r, "thread");
for(int id=0;id<idCount;id++) {
//System.out.println("TEST: check id=" + id + " expectedThreadID=" + expectedThreadIDs[id]);
TopDocs hits = s.search(new TermQuery(new Term("id", ""+id)), 1);
if (expectedThreadIDs[id] != -1) {
assertEquals(1, hits.totalHits);
int actualThreadID = (int) docValues.get(id);
if (expectedThreadIDs[id] != actualThreadID) {
System.out.println("FAIL: id=" + id + " expectedThreadID=" + expectedThreadIDs[id] + " vs actualThreadID=" + actualThreadID + " commitSeqNo=" + commitSeqNo + " numThreads=" + numThreads);
for(int threadID=0;threadID<threadOps.size();threadID++) {
for(Operation op : threadOps.get(threadID)) {
if (id == op.id) {
System.out.println(" threadID=" + threadID + " seqNo=" + op.seqNo + " " + (op.what == 2 ? "updated" : "deleted"));
}
}
}
assertEquals("id=" + id, expectedThreadIDs[id], actualThreadID);
}
} else if (hits.totalHits != 0) {
System.out.println("FAIL: id=" + id + " expectedThreadID=" + expectedThreadIDs[id] + " vs totalHits=" + hits.totalHits + " commitSeqNo=" + commitSeqNo + " numThreads=" + numThreads);
for(int threadID=0;threadID<threadOps.size();threadID++) {
for(Operation op : threadOps.get(threadID)) {
if (id == op.id) {
@ -347,7 +517,7 @@ public class TestIndexingSequenceNumbers extends LuceneTestCase {
}
} else {
Document doc = new Document();
doc.add(new StoredField("thread", threadID));
doc.add(new StoredField("threadop", threadID + "-" + ops.size()));
doc.add(new StringField("id", "" + op.id, Field.Store.NO));
if (random().nextBoolean()) {
List<Document> docs = new ArrayList<>();
@ -366,6 +536,7 @@ public class TestIndexingSequenceNumbers extends LuceneTestCase {
}
}
};
threads[i].setName("thread" + threadID);
threads[i].start();
}
startingGun.countDown();
@ -422,7 +593,34 @@ public class TestIndexingSequenceNumbers extends LuceneTestCase {
for(int id=0;id<idCount;id++) {
//System.out.println("TEST: check id=" + id + " expectedThreadID=" + expectedThreadIDs[id]);
assertEquals(expectedCounts[id], s.count(new TermQuery(new Term("id", ""+id))));
int actualCount = s.count(new TermQuery(new Term("id", ""+id)));
if (expectedCounts[id] != actualCount) {
System.out.println("TEST: FAIL r=" + r + " id=" + id + " commitSeqNo=" + commitSeqNo);
for(int threadID=0;threadID<threadOps.size();threadID++) {
int opCount2 = 0;
for(Operation op : threadOps.get(threadID)) {
if (op.id == id) {
boolean shouldCount = op.seqNo <= commitSeqNo && op.seqNo > lastDelSeqNos[op.id];
System.out.println(" id=" + id + " what=" + op.what + " threadop=" + threadID + "-" + opCount2 + " seqNo=" + op.seqNo + " vs lastDelSeqNo=" + lastDelSeqNos[op.id] + " shouldCount=" + shouldCount);
}
opCount2++;
}
}
TopDocs hits = s.search(new TermQuery(new Term("id", ""+id)), 1+actualCount);
for(ScoreDoc hit : hits.scoreDocs) {
System.out.println(" hit: " + s.doc(hit.doc).get("threadop"));
}
for(LeafReaderContext ctx : r.leaves()) {
System.out.println(" sub=" + ctx.reader());
Bits liveDocs = ctx.reader().getLiveDocs();
for(int docID=0;docID<ctx.reader().maxDoc();docID++) {
System.out.println(" docID=" + docID + " threadop=" + ctx.reader().document(docID).get("threadop") + (liveDocs != null && liveDocs.get(docID) == false ? " (deleted)" : ""));
}
}
assertEquals("commit " + i + " of " + commits.size() + " id=" + id + " reader=" + r, expectedCounts[id], actualCount);
}
}
w.close();
r.close();

112
lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
View File

@ -18,6 +18,8 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.HashSet;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
@ -58,12 +60,118 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
public class TestNumericDocValuesUpdates extends LuceneTestCase {
private Document doc(int id) {
// make sure we don't set the doc's value to 0, to not confuse with a document that's missing values
return doc(id, id +1);
}
private Document doc(int id, long val) {
Document doc = new Document();
doc.add(new StringField("id", "doc-" + id, Store.NO));
// make sure we don't set the doc's value to 0, to not confuse with a document that's missing values
doc.add(new NumericDocValuesField("val", id + 1));
doc.add(new NumericDocValuesField("val", val));
return doc;
}
public void testMultipleUpdatesSameDoc() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
conf.setMaxBufferedDocs(3); // small number of docs, so use a tiny maxBufferedDocs
IndexWriter writer = new IndexWriter(dir, conf);
writer.updateDocument (new Term("id","doc-1"), doc(1, 1000000000L ));
writer.updateNumericDocValue(new Term("id","doc-1"), "val", 1000001111L );
writer.updateDocument (new Term("id","doc-2"), doc(2, 2000000000L ));
writer.updateDocument (new Term("id","doc-2"), doc(2, 2222222222L ));
writer.updateNumericDocValue(new Term("id","doc-1"), "val", 1111111111L );
writer.commit();
final DirectoryReader reader = DirectoryReader.open(dir);
final IndexSearcher searcher = new IndexSearcher(reader);
TopFieldDocs td;
td = searcher.search(new TermQuery(new Term("id", "doc-1")), 1,
new Sort(new SortField("val", SortField.Type.LONG)));
assertEquals("doc-1 missing?", 1, td.scoreDocs.length);
assertEquals("doc-1 value", 1111111111L, ((FieldDoc)td.scoreDocs[0]).fields[0]);
td = searcher.search(new TermQuery(new Term("id", "doc-2")), 1,
new Sort(new SortField("val", SortField.Type.LONG)));
assertEquals("doc-2 missing?", 1, td.scoreDocs.length);
assertEquals("doc-2 value", 2222222222L, ((FieldDoc)td.scoreDocs[0]).fields[0]);
IOUtils.close(reader, writer, dir);
}
public void testBiasedMixOfRandomUpdates() throws Exception {
// 3 types of operations: add, updated, updateDV.
// rather then randomizing equally, we'll pick (random) cutoffs so each test run is biased,
// in terms of some ops happen more often then others
final int ADD_CUTOFF = TestUtil.nextInt(random(), 1, 98);
final int UPD_CUTOFF = TestUtil.nextInt(random(), ADD_CUTOFF+1, 99);
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
IndexWriter writer = new IndexWriter(dir, conf);
final int numOperations = atLeast(1000);
final Map<Integer,Long> expected = new HashMap<>(numOperations / 3);
// start with at least one doc before any chance of updates
final int numSeedDocs = atLeast(1);
for (int i = 0; i < numSeedDocs; i++) {
final long val = random().nextLong();
expected.put(i, val);
writer.addDocument(doc(i, val));
}
int numDocUpdates = 0;
int numValueUpdates = 0;
//System.out.println("TEST: numOperations=" + numOperations + " ADD_CUTOFF=" + ADD_CUTOFF + " UPD_CUTOFF=" + UPD_CUTOFF);
for (int i = 0; i < numOperations; i++) {
final int op = TestUtil.nextInt(random(), 1, 100);
final long val = random().nextLong();
if (op <= ADD_CUTOFF) {
final int id = expected.size();
//System.out.println("TEST i=" + i + ": addDocument id=" + id + " val=" + val);
expected.put(id, val);
writer.addDocument(doc(id, val));
} else {
final int id = TestUtil.nextInt(random(), 0, expected.size()-1);
expected.put(id, val);
if (op <= UPD_CUTOFF) {
numDocUpdates++;
//System.out.println("TEST i=" + i + ": updateDocument id=" + id + " val=" + val);
writer.updateDocument(new Term("id","doc-" + id), doc(id, val));
} else {
numValueUpdates++;
//System.out.println("TEST i=" + i + ": updateDV id=" + id + " val=" + val);
writer.updateNumericDocValue(new Term("id","doc-" + id), "val", val);
}
}
}
writer.commit();
final DirectoryReader reader = DirectoryReader.open(dir);
final IndexSearcher searcher = new IndexSearcher(reader);
// TODO: make more efficient if max numOperations is going to be increased much
for (Map.Entry<Integer,Long> expect : expected.entrySet()) {
String id = "doc-" + expect.getKey();
TopFieldDocs td = searcher.search(new TermQuery(new Term("id", id)), 1,
new Sort(new SortField("val", SortField.Type.LONG)));
assertEquals(id + " missing?", 1, td.totalHits);
assertEquals(id + " value", expect.getValue(), ((FieldDoc)td.scoreDocs[0]).fields[0]);
}
IOUtils.close(reader, writer, dir);
}
@Test
public void testUpdatesAreFlushed() throws IOException {

11
lucene/core/src/test/org/apache/lucene/search/spans/TestSpanCollection.java
View File

@ -17,13 +17,16 @@
package org.apache.lucene.search.spans;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@ -32,10 +35,6 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
public class TestSpanCollection extends LuceneTestCase {
protected IndexSearcher searcher;
@ -61,7 +60,7 @@ public class TestSpanCollection extends LuceneTestCase {
super.setUp();
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
newIndexWriterConfig(new MockAnalyzer(random())));
newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
for (int i = 0; i < docFields.length; i++) {
Document doc = new Document();
doc.add(newField(FIELD, docFields[i], OFFSETS));

36
lucene/core/src/test/org/apache/lucene/util/TestArrayUtil.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.util;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Random;
public class TestArrayUtil extends LuceneTestCase {
@ -275,5 +276,38 @@ public class TestArrayUtil extends LuceneTestCase {
ArrayUtil.introSort(a, Collections.reverseOrder());
ArrayUtil.timSort(a, Collections.reverseOrder());
}
public void testSelect() {
for (int iter = 0; iter < 100; ++iter) {
doTestSelect();
}
}
private void doTestSelect() {
final int from = random().nextInt(5);
final int to = from + TestUtil.nextInt(random(), 1, 10000);
final int max = random().nextBoolean() ? random().nextInt(100) : random().nextInt(100000);
Integer[] arr = new Integer[from + to + random().nextInt(5)];
for (int i = 0; i < arr.length; ++i) {
arr[i] = TestUtil.nextInt(random(), 0, max);
}
final int k = TestUtil.nextInt(random(), from, to - 1);
Integer[] expected = arr.clone();
Arrays.sort(expected, from, to);
Integer[] actual = arr.clone();
ArrayUtil.select(actual, from, to, k, Comparator.naturalOrder());
assertEquals(expected[k], actual[k]);
for (int i = 0; i < actual.length; ++i) {
if (i < from || i >= to) {
assertSame(arr[i], actual[i]);
} else if (i <= k) {
assertTrue(actual[i].intValue() <= actual[k].intValue());
} else {
assertTrue(actual[i].intValue() >= actual[k].intValue());
}
}
}
}

3
lucene/misc/src/test/org/apache/lucene/store/TestHardLinkCopyDirectoryWrapper.java
View File

@ -21,6 +21,7 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collections;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.util.IOUtils;
@ -51,6 +52,8 @@ public class TestHardLinkCopyDirectoryWrapper extends BaseDirectoryTestCase {
output.writeString("hey man, nice shot!");
CodecUtil.writeFooter(output);
}
// In case luceneDir_1 has an NRTCachingDirectory
luceneDir_1.sync(Collections.singleton("foo.bar"));
try {
Files.createLink(tempDir.resolve("test"), dir_1.resolve("foo.bar"));
BasicFileAttributes destAttr = Files.readAttributes(tempDir.resolve("test"), BasicFileAttributes.class);

2
lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/XYZBounds.java
View File

@ -30,7 +30,7 @@ public class XYZBounds implements Bounds {
* unacceptably large.
* Also, see LUCENE-7290 for a description of how geometry can magnify the bounds delta.
*/
private static final double FUDGE_FACTOR = Vector.MINIMUM_RESOLUTION * 500.0;
private static final double FUDGE_FACTOR = Vector.MINIMUM_RESOLUTION * 1000.0;
/** Minimum x */
private Double minX = null;

8
solr/CHANGES.txt
View File

@ -148,6 +148,8 @@ New Features
* SOLR-8583: Apply highlighting to hl.alternateField by default for Default and FastVectorHighlighter.
Turn off with hl.highlightAlternate=false (janhoy, David Smiley)
* SOLR-7123: '/update/json/docs' path supports nested documents (noble)
Bug Fixes
----------------------
@ -245,6 +247,8 @@ Bug Fixes
* SOLR-9165: Spellcheck does not return collations if "maxCollationTries" is used with "cursorMark".
(James Dyer)
* SOLR-8940: Fix group.sort option (hossman)
Optimizations
----------------------
* SOLR-8722: Don't force a full ZkStateReader refresh on every Overseer operation.
@ -340,11 +344,11 @@ Other Changes
* SOLR-9110: Move JoinFromCollection- SubQueryTransformer- BlockJoinFacet- Distrib Tests to SolrCloudTestCase (Mikhail Khludnev)
* SOLR-9161: SolrPluginUtils.invokeSetters now accommodates setter variants (Christine Poerschke)
* SOLR-9136: Separate out the error statistics into server-side error vs client-side error
(Jessica Cheng Mallet via Erick Erickson)
* SOLR-9107: new @RandomizeSSL annotation for more fine grained control of SSL testing (hossman, sarowe)
================== 6.0.1 ==================
(No Changes)

37
solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
View File

@ -64,7 +64,7 @@ import static org.apache.solr.common.params.CommonParams.PATH;
*/
public class JsonLoader extends ContentStreamLoader {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String CHILD_DOC_KEY = "_childDocuments_";
public static final String CHILD_DOC_KEY = "_childDocuments_";
@Override
public String getDefaultWT() {
@ -125,8 +125,9 @@ public class JsonLoader extends ContentStreamLoader {
String path = (String) req.getContext().get(PATH);
if (UpdateRequestHandler.DOC_PATH.equals(path) || "false".equals(req.getParams().get("json.command"))) {
String split = req.getParams().get("split");
String childSplit = req.getParams().get("child.split");
String[] f = req.getParams().getParams("f");
handleSplitMode(split, f, reader);
handleSplitMode(split, childSplit, f, reader);
return;
}
parser = new JSONParser(reader);
@ -193,7 +194,7 @@ public class JsonLoader extends ContentStreamLoader {
}
}
private void handleSplitMode(String split, String[] fields, final Reader reader) throws IOException {
private void handleSplitMode(String split, String childSplit, String[] fields, final Reader reader) throws IOException {
if (split == null) split = "/";
if (fields == null || fields.length == 0) fields = new String[]{"$FQN:/**"};
final boolean echo = "true".equals(req.getParams().get("echo"));
@ -208,7 +209,7 @@ public class JsonLoader extends ContentStreamLoader {
}
JsonRecordReader jsonRecordReader = JsonRecordReader.getInst(split, Arrays.asList(fields));
JsonRecordReader jsonRecordReader = JsonRecordReader.getInst(split, childSplit, Arrays.asList(fields));
jsonRecordReader.streamRecords(parser, new JsonRecordReader.Handler() {
ArrayList docs = null;
@ -221,15 +222,16 @@ public class JsonLoader extends ContentStreamLoader {
docs = new ArrayList();
rsp.add("docs", docs);
}
if (copy.containsKey(null)) {
copy.put(CHILD_DOC_KEY, copy.get(null));
copy.remove(null);
}
docs.add(copy);
} else {
AddUpdateCommand cmd = new AddUpdateCommand(req);
cmd.commitWithin = commitWithin;
cmd.overwrite = overwrite;
cmd.solrDoc = new SolrInputDocument();
for (Map.Entry<String, Object> entry : copy.entrySet()) {
cmd.solrDoc.setField(entry.getKey(), entry.getValue());
}
cmd.solrDoc = buildDoc(copy);
try {
processor.processAdd(cmd);
} catch (IOException e) {
@ -240,6 +242,25 @@ public class JsonLoader extends ContentStreamLoader {
});
}
private SolrInputDocument buildDoc(Map<String, Object> m) {
SolrInputDocument result = new SolrInputDocument();
for (Map.Entry<String, Object> e : m.entrySet()) {
if (e.getKey() == null) {// special case. JsonRecordReader emits child docs with null key
if (e.getValue() instanceof List) {
List value = (List) e.getValue();
for (Object o : value) {
if (o instanceof Map) result.addChildDocument(buildDoc((Map) o));
}
} else if (e.getValue() instanceof Map) {
result.addChildDocument(buildDoc((Map) e));
}
} else {
result.setField(e.getKey(), e.getValue());
}
}
return result;
}
private Map<String, Object> getDocMap(Map<String, Object> record, JSONParser parser, String srcField, boolean mapUniqueKeyOnly) {
Map result = record;
if (srcField != null && parser instanceof RecordingJSONParser) {

2
solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
View File

@ -134,7 +134,7 @@ public class TopGroupsResultTransformer implements ShardResultTransformer<List<C
@SuppressWarnings("unchecked")
List<NamedList<Object>> documents = (List<NamedList<Object>>) groupResult.get("documents");
ScoreDoc[] scoreDocs = transformToNativeShardDoc(documents, groupSort, shard, schema);
ScoreDoc[] scoreDocs = transformToNativeShardDoc(documents, sortWithinGroup, shard, schema);
BytesRef groupValueRef = groupValue != null ? new BytesRef(groupValue) : null;
groupDocs.add(new GroupDocs<>(Float.NaN, maxScore, totalGroupHits, scoreDocs, groupValueRef, null));

14
solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
View File

@ -1066,8 +1066,8 @@ public class SolrPluginUtils {
String key = entry.getKey();
String setterName = "set" + String.valueOf(Character.toUpperCase(key.charAt(0))) + key.substring(1);
try {
final Method method = findSetter(clazz, setterName, key);
final Object val = entry.getValue();
final Method method = findSetter(clazz, setterName, key, val.getClass());
method.invoke(bean, val);
} catch (InvocationTargetException | IllegalAccessException e1) {
throw new RuntimeException("Error invoking setter " + setterName + " on class : " + clazz.getName(), e1);
@ -1075,14 +1075,10 @@ public class SolrPluginUtils {
}
}
private static Method findSetter(Class<?> clazz, String setterName, String key, Class<?> paramClazz) {
try {
return clazz.getMethod(setterName, new Class<?>[] { paramClazz });
} catch (NoSuchMethodException e) {
for (Method m : clazz.getMethods()) {
if (m.getName().equals(setterName) && m.getParameterTypes().length == 1) {
return m;
}
private static Method findSetter(Class<?> clazz, String setterName, String key) {
for (Method m : clazz.getMethods()) {
if (m.getName().equals(setterName) && m.getParameterTypes().length == 1) {
return m;
}
}
throw new RuntimeException("No setter corrresponding to '" + key + "' in " + clazz.getName());

34
solr/core/src/test/org/apache/solr/TestDistributedGrouping.java
View File

@ -23,9 +23,11 @@ import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.SolrDocumentList;
import org.junit.Test;
import java.io.IOException;
import java.util.List;
/**
* TODO? perhaps use:
@ -44,7 +46,7 @@ public class TestDistributedGrouping extends BaseDistributedSearchTestCase {
String tlong = "other_tl1";
String tdate_a = "a_n_tdt";
String tdate_b = "b_n_tdt";
String oddField="oddField_s";
String oddField="oddField_s1";
@Test
public void test() throws Exception {
@ -265,6 +267,36 @@ public class TestDistributedGrouping extends BaseDistributedSearchTestCase {
query("q", "{!func}id", "rows", 100, "fl", "score,id," + i1, "group", "true", "group.field", i1, "group.limit", 10, "sort", "score desc, _docid_ asc, id asc");
query("q", "{!func}id", "rows", 100, "fl", "score,id," + i1, "group", "true", "group.field", i1, "group.limit", 10);
// some explicit checks of non default sorting, and sort/group.sort with diff clauses
query("q", "{!func}id", "rows", 100, "fl", tlong + ",id," + i1, "group", "true",
"group.field", i1, "group.limit", 10,
"sort", tlong+" asc, id desc");
query("q", "{!func}id", "rows", 100, "fl", tlong + ",id," + i1, "group", "true",
"group.field", i1, "group.limit", 10,
"sort", "id asc",
"group.sort", tlong+" asc, id desc");
query("q", "{!func}id", "rows", 100, "fl", tlong + ",id," + i1, "group", "true",
"group.field", i1, "group.limit", 10,
"sort", tlong+" asc, id desc",
"group.sort", "id asc");
rsp = query("q", "{!func}id", "fq", oddField+":[* TO *]",
"rows", 100, "fl", tlong + ",id," + i1, "group", "true",
"group.field", i1, "group.limit", 10,
"sort", tlong+" asc",
"group.sort", oddField+" asc");
nl = (NamedList<?>) rsp.getResponse().get("grouped");
nl = (NamedList<?>) nl.get(i1);
assertEquals(rsp.toString(), 6, nl.get("matches"));
assertEquals(rsp.toString(), 2, ((List<NamedList<?>>)nl.get("groups")).size());
nl = ((List<NamedList<?>>)nl.get("groups")).get(0);
assertEquals(rsp.toString(), 232, nl.get("groupValue"));
SolrDocumentList docs = (SolrDocumentList) nl.get("doclist");
assertEquals(docs.toString(), 5, docs.getNumFound());
assertEquals(docs.toString(), 22, docs.get(0).getFirstValue("id"));
assertEquals(docs.toString(), 21, docs.get(4).getFirstValue("id"));
// Can't validate the response, but can check if no errors occur.
simpleQuery("q", "*:*", "rows", 100, "fl", "id," + i1, "group", "true", "group.query", t1 + ":kings OR " + t1 + ":eggs", "group.limit", 10, "sort", i1 + " asc, id asc", CommonParams.TIME_ALLOWED, 1);

202
solr/core/src/test/org/apache/solr/cloud/TestSSLRandomization.java
View File

@ -17,9 +17,13 @@
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.util.Arrays;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.util.SSLTestConfig;
import org.apache.solr.util.RandomizeSSL;
import org.apache.solr.util.RandomizeSSL.SSLRandomizer;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@ -34,6 +38,7 @@ import org.slf4j.LoggerFactory;
*
* @see TestMiniSolrCloudClusterSSL
*/
@RandomizeSSL(ssl=0.5,reason="frequent SSL usage to make test worth while")
public class TestSSLRandomization extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ -51,4 +56,201 @@ public class TestSSLRandomization extends SolrCloudTestCase {
String url = buildUrl(6666, "/foo");
assertEquals(sslConfig.isSSLMode() ? "https://127.0.0.1:6666/foo" : "http://127.0.0.1:6666/foo", url);
}
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(ssl=0.42,clientAuth=0.33,reason="foo")
public class FullyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
public class InheritedFullyAnnotated extends FullyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
public class NotAnnotated { };
/** Used by {@link #testSSLRandomizer} */
public class InheritedNotAnnotated extends NotAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@SuppressSSL(bugUrl="fakeBugUrl")
public class Suppressed { };
/** Used by {@link #testSSLRandomizer} */
public class InheritedSuppressed extends Suppressed { };
/** Used by {@link #testSSLRandomizer} */
@SuppressSSL(bugUrl="fakeBugUrl")
public class InheritedAnnotationButSuppressed extends FullyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(ssl=0.42,clientAuth=0.33,reason="foo")
public class InheritedSuppressedWithIgnoredAnnotation extends Suppressed {
// Even with direct annotation, supression at superclass overrules us.
//
// (If it didn't work this way, it would be a pain in the ass to quickly disable SSL for a
// broad hierarchy of tests)
};
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL()
public class EmptyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
public class InheritedEmptyAnnotated extends EmptyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(0.5)
public class InheritedEmptyAnnotatationWithOverride extends EmptyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(ssl=0.42,clientAuth=0.33,reason="foo")
public class GrandchildInheritedEmptyAnnotatationWithOverride extends InheritedEmptyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(0.5)
public class SimplyAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(0.0)
public class MinAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(1)
public class MaxAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(ssl=0.42)
public class SSlButNoClientAuthAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(clientAuth=0.42)
public class ClientAuthButNoSSLAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(ssl=42.0)
public class SSLOutOfRangeAnnotated { };
/** Used by {@link #testSSLRandomizer} */
@RandomizeSSL(clientAuth=42.0)
public class ClientAuthOutOfRangeAnnotated { };
/** Used by {@link #testSSLRandomizer} */
public class InheritedOutOfRangeAnnotated extends ClientAuthOutOfRangeAnnotated { };
public void testSSLRandomizer() {
SSLRandomizer r;
// for some cases, we know exactly what the config should be regardless of randomization factors
SSLTestConfig conf;
for (Class c : Arrays.asList(FullyAnnotated.class, InheritedFullyAnnotated.class,
GrandchildInheritedEmptyAnnotatationWithOverride.class )) {
r = SSLRandomizer.getSSLRandomizerForClass(c);
assertEquals(c.toString(), 0.42D, r.ssl, 0.0D);
assertEquals(c.toString(), 0.33D, r.clientAuth, 0.0D);
assertTrue(c.toString(), r.debug.contains("foo"));
}
for (Class c : Arrays.asList(NotAnnotated.class, InheritedNotAnnotated.class)) {
r = SSLRandomizer.getSSLRandomizerForClass(c);
assertEquals(c.toString(), 0.0D, r.ssl, 0.0D);
assertEquals(c.toString(), 0.0D, r.clientAuth, 0.0D);
assertTrue(c.toString(), r.debug.contains("not specified"));
conf = r.createSSLTestConfig();
assertEquals(c.toString(), false, conf.isSSLMode());
assertEquals(c.toString(), false, conf.isClientAuthMode());
}
for (Class c : Arrays.asList(Suppressed.class,
InheritedSuppressed.class,
InheritedAnnotationButSuppressed.class,
InheritedSuppressedWithIgnoredAnnotation.class)) {
r = SSLRandomizer.getSSLRandomizerForClass(Suppressed.class);
assertEquals(c.toString(), 0.0D, r.ssl, 0.0D);
assertEquals(c.toString(), 0.0D, r.clientAuth, 0.0D);
assertTrue(c.toString(), r.debug.contains("SuppressSSL"));
assertTrue(c.toString(), r.debug.contains("fakeBugUrl"));
conf = r.createSSLTestConfig();
assertEquals(c.toString(), false, conf.isSSLMode());
assertEquals(c.toString(), false, conf.isClientAuthMode());
}
for (Class c : Arrays.asList(EmptyAnnotated.class, InheritedEmptyAnnotated.class)) {
r = SSLRandomizer.getSSLRandomizerForClass(c);
assertEquals(c.toString(), RandomizeSSL.DEFAULT_ODDS, r.ssl, 0.0D);
assertEquals(c.toString(), RandomizeSSL.DEFAULT_ODDS, r.clientAuth, 0.0D);
}
for (Class c : Arrays.asList(SimplyAnnotated.class, InheritedEmptyAnnotatationWithOverride.class)) {
r = SSLRandomizer.getSSLRandomizerForClass(c);
assertEquals(c.toString(), 0.5D, r.ssl, 0.0D);
assertEquals(c.toString(), 0.5D, r.clientAuth, 0.0D);
}
r = SSLRandomizer.getSSLRandomizerForClass(MinAnnotated.class);
assertEquals(0.0D, r.ssl, 0.0D);
assertEquals(0.0D, r.clientAuth, 0.0D);
conf = r.createSSLTestConfig();
assertEquals(false, conf.isSSLMode());
assertEquals(false, conf.isClientAuthMode());
r = SSLRandomizer.getSSLRandomizerForClass(MaxAnnotated.class);
assertEquals(1.0D, r.ssl, 0.0D);
assertEquals(1.0D, r.clientAuth, 0.0D);
conf = r.createSSLTestConfig();
assertEquals(true, conf.isSSLMode());
assertEquals(true, conf.isClientAuthMode());
r = SSLRandomizer.getSSLRandomizerForClass(SSlButNoClientAuthAnnotated.class);
assertEquals(0.42D, r.ssl, 0.0D);
assertEquals(0.42D, r.clientAuth, 0.0D);
r = SSLRandomizer.getSSLRandomizerForClass(ClientAuthButNoSSLAnnotated.class);
assertEquals(RandomizeSSL.DEFAULT_ODDS, r.ssl, 0.0D);
assertEquals(0.42D, r.clientAuth, 0.0D);
for (Class c : Arrays.asList(SSLOutOfRangeAnnotated.class,
ClientAuthOutOfRangeAnnotated.class,
InheritedOutOfRangeAnnotated.class)) {
expectThrows(IllegalArgumentException.class, () -> {
Object trash = SSLRandomizer.getSSLRandomizerForClass(c);
});
}
}
public void testSSLRandomizerEffectiveOdds() {
assertEquals(RandomizeSSL.DEFAULT_ODDS,
SSLRandomizer.getEffectiveOdds(RandomizeSSL.DEFAULT_ODDS, false, 1), 0.0005D);
assertEquals(0.2727D,
SSLRandomizer.getEffectiveOdds(RandomizeSSL.DEFAULT_ODDS, true, 1), 0.0005D);
assertEquals(0.0100D, SSLRandomizer.getEffectiveOdds(0.01D, false, 1), 0.0005D);
assertEquals(0.1000D, SSLRandomizer.getEffectiveOdds(0.01D, true, 1), 0.0005D);
assertEquals(0.6206D, SSLRandomizer.getEffectiveOdds(0.01D, false, 5), 0.0005D);
assertEquals(0.5000D, SSLRandomizer.getEffectiveOdds(0.5D, false, 1), 0.0005D);
assertEquals(0.5454D, SSLRandomizer.getEffectiveOdds(0.5D, true, 1), 0.0005D);
assertEquals(0.8083D, SSLRandomizer.getEffectiveOdds(0.5D, false, 5), 0.0005D);
assertEquals(0.8000D, SSLRandomizer.getEffectiveOdds(0.8D, false, 1), 0.0005D);
assertEquals(0.8181D, SSLRandomizer.getEffectiveOdds(0.8D, true, 1), 0.0005D);
assertEquals(0.9233D, SSLRandomizer.getEffectiveOdds(0.8D, false, 5), 0.0005D);
// never ever
assertEquals(0.0D, SSLRandomizer.getEffectiveOdds(0.0D, false, 1), 0.0D);
assertEquals(0.0D, SSLRandomizer.getEffectiveOdds(0.0D, true, 100), 0.0D);
assertEquals(0.0D, SSLRandomizer.getEffectiveOdds(0.0D, false, 100), 0.0D);
assertEquals(0.0D, SSLRandomizer.getEffectiveOdds(0.0D, true, 10000), 0.0D);
assertEquals(0.0D, SSLRandomizer.getEffectiveOdds(0.0D, false, 10000), 0.0D);
assertEquals(0.0D, SSLRandomizer.getEffectiveOdds(0.0D, random().nextBoolean(), random().nextInt()), 0.0D);
// always
assertEquals(1.0D, SSLRandomizer.getEffectiveOdds(1.0D, false, 1), 0.0D);
assertEquals(1.0D, SSLRandomizer.getEffectiveOdds(1.0D, true, 100), 0.0D);
assertEquals(1.0D, SSLRandomizer.getEffectiveOdds(1.0D, false, 100), 0.0D);
assertEquals(1.0D, SSLRandomizer.getEffectiveOdds(1.0D, true, 10000), 0.0D);
assertEquals(1.0D, SSLRandomizer.getEffectiveOdds(1.0D, false, 10000), 0.0D);
assertEquals(1.0D, SSLRandomizer.getEffectiveOdds(1.0D, random().nextBoolean(), random().nextInt()), 0.0D);
}
}

21
solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
View File

@ -366,7 +366,28 @@ public class JsonLoaderTest extends SolrTestCaseJ4 {
obj = (Map) ObjectBuilder.fromJSON(content);
assertEquals("2", obj.get("id"));
String json = "{a:{" +
"b:[{c:c1, e:e1},{c:c2, e :e2, d:{p:q}}]," +
"x:y" +
"}}";
req = req("split", "/", "child.split" , "/a/b" );
req.getContext().put("path","/update/json/docs");
rsp = new SolrQueryResponse();
p = new BufferingRequestProcessor(null);
loader = new JsonLoader();
loader.load(req, rsp, new ContentStreamBase.StringStream(json), p);
assertEquals( 1, p.addCommands.size() );
assertEquals("y", p.addCommands.get(0).solrDoc.getFieldValue("a.x"));
List<SolrInputDocument> children = p.addCommands.get(0).solrDoc.getChildDocuments();
assertEquals(2, children.size());
SolrInputDocument d = children.get(0);
assertEquals(d.getFieldValue("c"), "c1");
assertEquals(d.getFieldValue("e"), "e1");
d = children.get(1);
assertEquals(d.getFieldValue("c"), "c2");
assertEquals(d.getFieldValue("e"), "e2");
assertEquals(d.getFieldValue("d.p"), "q");
}

28
solr/core/src/test/org/apache/solr/util/SolrPluginUtilsTest.java
View File

@ -455,34 +455,6 @@ public class SolrPluginUtilsTest extends SolrTestCaseJ4 {
assertEquals(3, q.build().getMinimumNumberShouldMatch());
}
private class InvokeSettersTestClass {
private float aFloat = random().nextFloat();
public float getAFloat() {
return aFloat;
}
public void setAFloat(float aFloat) {
this.aFloat = aFloat;
}
public void setAFloat(String aFloat) {
this.aFloat = Float.parseFloat(aFloat);
}
}
@Test
public void testInvokeSetters() {
final Float theFloat = new Float(random().nextFloat());
implTestInvokeSetters(theFloat, theFloat);
implTestInvokeSetters(theFloat, theFloat.toString());
}
public void implTestInvokeSetters(final Float theFloat, final Object theFloatObject) {
final InvokeSettersTestClass bean = new InvokeSettersTestClass();
final Map<String,Object> initArgs = new HashMap<>();
initArgs.put("aFloat", theFloatObject);
SolrPluginUtils.invokeSetters(bean, initArgs.entrySet());
assertEquals(bean.getAFloat(), theFloat.floatValue(), 0.0);
}
/** macro */
public String pe(CharSequence s) {
return SolrPluginUtils.partialEscape(s).toString();

138
solr/solrj/src/java/org/apache/solr/common/util/JsonRecordReader.java
View File

@ -34,9 +34,11 @@ public class JsonRecordReader {
private Node rootNode = new Node("/", (Node) null);
public static JsonRecordReader getInst(String split, List<String> fieldMappings) {
public static JsonRecordReader getInst(String split, String childSplit, List<String> fieldMappings) {
JsonRecordReader jsonRecordReader = new JsonRecordReader(split);
JsonRecordReader jsonRecordReader = new JsonRecordReader();
jsonRecordReader.addSplit(split);
if (childSplit != null) jsonRecordReader.addSplit(childSplit);
for (String s : fieldMappings) {
String path = s;
int idx = s.indexOf(':');
@ -50,14 +52,21 @@ public class JsonRecordReader {
return jsonRecordReader;
}
public static JsonRecordReader getInst(String split, List<String> fieldMappings) {
return getInst(split, null, fieldMappings);
}
private JsonRecordReader() {
}
/**
* A constructor called with a '|' separated list of path expressions
* a '|' separated list of path expressions
* which define sub sections of the JSON stream that are to be emitted as
* separate records.
* It is possible to have multiple levels of split one for parent and one for child
* each child record (or a list of records) will be emitted as a part of the parent record with
* null as the key
*
* @param splitPath The PATH for which a record is emitted. Once the
* path tag is encountered, the Node.getInst method starts collecting wanted
* fields and at the close of the tag, a record is emitted containing all
* @param splitPath The PATH for which a record is emitted. A record is emitted containing all
* fields collected since the tag start. Once
* emitted the collected fields are cleared. Any fields collected in the
* parent tag or above will also be included in the record, but these are
@ -65,7 +74,8 @@ public class JsonRecordReader {
* <p>
* It uses the ' | ' syntax of PATH to pass in multiple paths.
*/
private JsonRecordReader(String splitPath) {
void addSplit(String splitPath) {
String[] splits = splitPath.split("\\|");
for (String split : splits) {
split = split.trim();
@ -93,10 +103,10 @@ public class JsonRecordReader {
if (!path.startsWith("/")) throw new RuntimeException("All paths must start with '/' " + path);
List<String> paths = splitEscapeQuote(path);
if (paths.size() == 0) {
if (isRecord) rootNode.isRecord = true;
return;//the patrh is "/"
if (isRecord) rootNode.setAsRecord();
return;//the path is "/"
}
// deal with how split behaves when seperator starts a string!
// deal with how split behaves when separator starts with an empty string!
if ("".equals(paths.get(0).trim()))
paths.remove(0);
rootNode.build(paths, fieldName, multiValued, isRecord, path);
@ -113,11 +123,8 @@ public class JsonRecordReader {
*/
public List<Map<String, Object>> getAllRecords(Reader r) throws IOException {
final List<Map<String, Object>> results = new ArrayList<>();
streamRecords(r, new Handler() {
@Override
public void handle(Map<String, Object> record, String path) {
results.add(record);
}
streamRecords(r, (record, path) -> {
results.add(record);
});
return results;
}
@ -136,7 +143,7 @@ public class JsonRecordReader {
public void streamRecords(JSONParser parser, Handler handler) throws IOException {
rootNode.parse(parser, handler,
new LinkedHashMap<String, Object>());
new LinkedHashMap<>());
}
@ -158,6 +165,7 @@ public class JsonRecordReader {
Node parent; // parent Node in the tree
boolean isLeaf = false; // flag: store/emit streamed text for this node
boolean isRecord = false; //flag: this Node starts a new record
boolean isChildRecord = false;
Node wildCardChild;
Node recursiveWildCardChild;
private boolean useFqn = false;
@ -176,9 +184,27 @@ public class JsonRecordReader {
this.fieldName = fieldName; // name to store collected values against
}
void setAsRecord() {
if (isMyChildARecord()) throw new RuntimeException(name + " has a parent node at my level or lower");
isChildRecord = hasParentRecord();
isRecord = true;
}
private boolean hasParentRecord() {
return isRecord || parent != null && parent.hasParentRecord();
}
private boolean isMyChildARecord() {
if (isRecord) return true;
for (Node node : childNodes.values()) {
if (node.isMyChildARecord()) return true;
}
return false;
}
/**
* Walk the Node tree propagating any wildDescentant information to
* Walk the Node tree propagating any wild Descendant information to
* child nodes.
*/
private void buildOptimize() {
@ -191,7 +217,7 @@ public class JsonRecordReader {
static final String RECURSIVE_WILDCARD_PATH = "**";
/**
* Build a Node tree structure representing all paths of intrest to us.
* Build a Node tree structure representing all paths of interest to us.
* This must be done before parsing of the JSON stream starts. Each node
* holds one portion of an path. Taking each path segment in turn this
* method walks the Node tree and finds where the new segment should be
@ -214,7 +240,7 @@ public class JsonRecordReader {
if (paths.isEmpty()) {
// We have emptied paths, we are for the moment a leaf of the tree.
// When parsing the actual input we have traversed to a position
// where we actutally have to do something. getOrAddNode() will
// where we actually have to do something. getOrAddNode() will
// have created and returned a new minimal Node with name and
// pathName already populated. We need to add more information.
if (record) {
@ -222,7 +248,7 @@ public class JsonRecordReader {
assert !WILDCARD_PATH.equals(n.name);
assert !RECURSIVE_WILDCARD_PATH.equals(n.name);
// split attribute
n.isRecord = true; // flag: split attribute, prepare to emit rec
n.setAsRecord(); // flag: split attribute, prepare to emit rec
n.splitPath = fieldName; // the full split attribute path
} else {
if (n.name.equals(WILDCARD_PATH)) {
@ -284,13 +310,13 @@ public class JsonRecordReader {
event = parser.nextEvent();
if (event == EOF) break;
if (event == OBJECT_START) {
handleObjectStart(parser, handler, values, new Stack<Set<String>>(), recordStarted, null);
handleObjectStart(parser, handler, values, new Stack<>(), recordStarted, null);
} else if (event == ARRAY_START) {
for (; ; ) {
event = parser.nextEvent();
if (event == ARRAY_END) break;
if (event == OBJECT_START) {
handleObjectStart(parser, handler, values, new Stack<Set<String>>(), recordStarted, null);
handleObjectStart(parser, handler, values, new Stack<>(), recordStarted, null);
}
}
}
@ -300,14 +326,14 @@ public class JsonRecordReader {
/**
* If a new tag is encountered, check if it is of interest or not by seeing
* if it matches against our node tree. If we have deperted from the node
* if it matches against our node tree. If we have departed from the node
* tree then walk back though the tree's ancestor nodes checking to see if
* any // expressions exist for the node and compare them against the new
* tag. If matched then "jump" to that node, otherwise ignore the tag.
* <p>
* Note, the list of // expressions found while walking back up the tree
* is chached in the HashMap decends. Then if the new tag is to be skipped,
* any inner chil tags are compared against the cache and jumped to if
* is cached in the HashMap descendants. Then if the new tag is to be skipped,
* any inner child tags are compared against the cache and jumped to if
* matched.
*/
private void handleObjectStart(final JSONParser parser,
@ -320,7 +346,7 @@ public class JsonRecordReader {
Set<String> valuesAddedinThisFrame = null;
if (isRecord || !recordStarted) {
// This Node is a match for an PATH from a forEach attribute,
// prepare for the clean up that will occurr when the record
// prepare for the clean up that will occur when the record
// is emitted after its END_ELEMENT is matched
valuesAddedinThisFrame = new HashSet<>();
stack.push(valuesAddedinThisFrame);
@ -340,17 +366,30 @@ public class JsonRecordReader {
@Override
public void walk(int event) throws IOException {
if (event == OBJECT_START) {
node.handleObjectStart(parser, handler, values, stack, isRecordStarted, this);
walkObject();
} else if (event == ARRAY_START) {
for (; ; ) {
event = parser.nextEvent();
if (event == ARRAY_END) break;
if (event == OBJECT_START) {
node.handleObjectStart(parser, handler, values, stack, isRecordStarted, this);
walkObject();
}
}
}
}
void walkObject() throws IOException {
if (node.isChildRecord) {
node.handleObjectStart(parser,
(record, path) -> addChildDoc2ParentDoc(record, values),
new LinkedHashMap<>(),
new Stack<>(),
true,
this
);
} else {
node.handleObjectStart(parser, handler, values, stack, isRecordStarted, this);
}
}
}
@ -372,7 +411,7 @@ public class JsonRecordReader {
if (node == null) node = recursiveWildCardChild;
if (node != null) {
if (node.isLeaf) {//this is a leaf collect data here
if (node.isLeaf) {//this is a leaf. Collect data here
event = parser.nextEvent();
String nameInRecord = node.fieldName == null ? getNameInRecord(name, frameWrapper, node) : node.fieldName;
MethodFrameWrapper runnable = null;
@ -390,7 +429,7 @@ public class JsonRecordReader {
new Wrapper(node, frameWrapper, name).walk(event);
}
} else {
//this is not something we are interested in . skip it
//this is not something we are interested in. Skip it
event = parser.nextEvent();
if (event == STRING ||
event == LONG ||
@ -420,10 +459,27 @@ public class JsonRecordReader {
}
}
private void addChildDoc2ParentDoc(Map<String, Object> record, Map<String, Object> values) {
Object oldVal = values.get(null);
if (oldVal == null) {
values.put(null, record);
} else if (oldVal instanceof List) {
((List) oldVal).add(record);
} else {
ArrayList l = new ArrayList();
l.add(oldVal);
l.add(record);
values.put(null, l);
}
}
/**
* Construct the name as it would appear in the final record
*/
private String getNameInRecord(String name, MethodFrameWrapper frameWrapper, Node n) {
if (frameWrapper == null || !n.useFqn) return name;
if (frameWrapper == null || !n.useFqn || frameWrapper.node.isChildRecord) return name;
StringBuilder sb = new StringBuilder();
frameWrapper.prependName(sb);
frameWrapper.addName(sb);
return sb.append(DELIM).append(name).toString();
}
@ -459,19 +515,19 @@ public class JsonRecordReader {
/**
* The path is split into segments using the '/' as a seperator. However
* The path is split into segments using the '/' as a separator. However
* this method deals with special cases where there is a slash '/' character
* inside the attribute value e.g. x/@html='text/html'. We split by '/' but
* then reassemble things were the '/' appears within a quoted sub-string.
* <p>
* We have already enforced that the string must begin with a seperator. This
* We have already enforced that the string must begin with a separator. This
* method depends heavily on how split behaves if the string starts with the
* seperator or if a sequence of multiple seperator's appear.
* seperator or if a sequence of multiple separators appear.
*/
private static List<String> splitEscapeQuote(String str) {
List<String> result = new LinkedList<>();
String[] ss = str.split("/");
for (int i = 0; i < ss.length; i++) { // i=1: skip seperator at start of string
for (int i = 0; i < ss.length; i++) { // i=1: skip separator at start of string
StringBuilder sb = new StringBuilder();
int quoteCount = 0;
while (true) {
@ -493,7 +549,7 @@ public class JsonRecordReader {
/**
* Implement this interface to stream records as and when one is found.
*/
public static interface Handler {
public interface Handler {
/**
* @param record The record map. The key is the field name as provided in
* the addField() methods. The value can be a single String (for single
@ -502,7 +558,7 @@ public class JsonRecordReader {
* If there is any change all parsing will be aborted and the Exception
* is propagated up
*/
public void handle(Map<String, Object> record, String path);
void handle(Map<String, Object> record, String path);
}
public static Object parseSingleFieldValue(int ev, JSONParser parser, MethodFrameWrapper runnable) throws IOException {
@ -539,9 +595,9 @@ public class JsonRecordReader {
MethodFrameWrapper parent;
String name;
void prependName(StringBuilder sb) {
if (parent != null) {
parent.prependName(sb);
void addName(StringBuilder sb) {
if (parent != null && !parent.node.isChildRecord) {
parent.addName(sb);
sb.append(DELIM);
}
sb.append(name);

62
solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
View File

@ -17,6 +17,7 @@
package org.apache.solr.common.util;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.handler.loader.JsonLoader;
import org.apache.solr.util.RecordingJSONParser;
import java.io.IOException;
@ -202,6 +203,44 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 {
}
public void testNestedDocs() throws Exception {
String json = "{a:{" +
"b:{c:d}," +
"x:y" +
"}}";
JsonRecordReader streamer = JsonRecordReader.getInst("/", "/a/b", Arrays.asList("/a/x", "/a/b/*"));
streamer.streamRecords(new StringReader(json), (record, path) -> {
assertEquals(record.get("x"), "y");
assertEquals(((Map) record.get(null)).get("c"), "d");
});
json = "{a:{" +
"b:[{c:c1, e:e1},{c:c2, e :e2, d:{p:q}}]," +
"x:y" +
"}}";
streamer.streamRecords(new StringReader(json), (record, path) -> {
assertEquals(record.get("x"), "y");
List l = (List) record.get(null);
Map m = (Map) l.get(0);
assertEquals(m.get("c"), "c1");
assertEquals(m.get("e"), "e1");
m = (Map) l.get(1);
assertEquals(m.get("c"), "c2");
assertEquals(m.get("e"), "e2");
});
streamer = JsonRecordReader.getInst("/", "/a/b", Arrays.asList("$FQN:/**"));
streamer.streamRecords(new StringReader(json), (record, path) -> {
assertEquals(record.get("a.x"), "y");
List l = (List) record.get(null);
Map m = (Map) l.get(0);
assertEquals(m.get("c"), "c1");
assertEquals(m.get("e"), "e1");
m = (Map) l.get(1);
assertEquals(m.get("c"), "c2");
assertEquals(m.get("e"), "e2");
assertEquals(m.get("d.p"), "q");
});
}
public void testNestedJsonWithFloats() throws Exception {
String json = "{\n" +
@ -264,16 +303,13 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 {
final AtomicReference<WeakReference<String>> ref = new AtomicReference<>();
streamer = JsonRecordReader.getInst("/", Collections.singletonList("$FQN:/**"));
streamer.streamRecords(new StringReader(json), new JsonRecordReader.Handler() {
@Override
public void handle(Map<String, Object> record, String path) {
System.gc();
if (ref.get() != null) {
assertNull("This reference is still intact :" +ref.get().get() ,ref.get().get());
}
String fName = record.keySet().iterator().next();
ref.set(new WeakReference<String>(fName));
streamer.streamRecords(new StringReader(json), (record, path) -> {
System.gc();
if (ref.get() != null) {
assertNull("This reference is still intact :" + ref.get().get(), ref.get().get());
}
String fName = record.keySet().iterator().next();
ref.set(new WeakReference<>(fName));
});
@ -621,12 +657,8 @@ public class TestJsonRecordReader extends SolrTestCaseJ4 {
RecordingJSONParser parser = new RecordingJSONParser(new StringReader(json));
JsonRecordReader recordReader = JsonRecordReader.getInst("/",Collections.singletonList("/**"));
try {
recordReader.streamRecords(parser, new JsonRecordReader.Handler() {
@Override
public void handle(Map<String, Object> record, String path) {
/*don't care*/
}
});
recordReader.streamRecords(parser, (record, path) -> {
}); /*don't care*/
} catch (RuntimeException e) {
parser.error("").printStackTrace();
throw e;

27
solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
View File

@ -104,6 +104,8 @@ import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.servlet.DirectSolrConnection;
import org.apache.solr.util.AbstractSolrTestCase;
import org.apache.solr.util.RandomizeSSL;
import org.apache.solr.util.RandomizeSSL.SSLRandomizer;
import org.apache.solr.util.RefCounted;
import org.apache.solr.util.RevertDefaultThreadHandlerRule;
import org.apache.solr.util.SSLTestConfig;
@ -137,6 +139,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
})
@SuppressSysoutChecks(bugUrl = "Solr dumps tons of logs to console.")
@SuppressFileSystems("ExtrasFS") // might be ok, the failures with e.g. nightly runs might be "normal"
@RandomizeSSL()
public abstract class SolrTestCaseJ4 extends LuceneTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ -317,27 +320,21 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
}
private static SSLTestConfig buildSSLConfig() {
// test has been disabled
if (RandomizedContext.current().getTargetClass().isAnnotationPresent(SuppressSSL.class)) {
return new SSLTestConfig();
}
SSLRandomizer sslRandomizer =
SSLRandomizer.getSSLRandomizerForClass(RandomizedContext.current().getTargetClass());
// we don't choose ssl that often because of SOLR-5776
final boolean trySsl = random().nextInt(10) < 2;
// NOTE: clientAuth is useless unless trySsl==true, but we randomize it independently
// just in case it might find bugs in our test/ssl client code (ie: attempting to use
// SSL w/client cert to non-ssl servers)
boolean trySslClientAuth = random().nextInt(10) < 2;
if (Constants.MAC_OS_X) {
// see SOLR-9039
// If a solution is found to remove this, please make sure to also update
// TestMiniSolrCloudClusterSSL.testSslAndClientAuth as well.
trySslClientAuth = false;
sslRandomizer = new SSLRandomizer(sslRandomizer.ssl, 0.0D, (sslRandomizer.debug + " w/ MAC_OS_X supressed clientAuth"));
}
log.info("Randomized ssl ({}) and clientAuth ({})", trySsl, trySslClientAuth);
return new SSLTestConfig(trySsl, trySslClientAuth);
SSLTestConfig result = sslRandomizer.createSSLTestConfig();
log.info("Randomized ssl ({}) and clientAuth ({}) via: {}",
result.isSSLMode(), result.isClientAuthMode(), sslRandomizer.debug);
return result;
}
protected static JettyConfig buildJettyConfig(String context) {

174
solr/test-framework/src/java/org/apache/solr/util/RandomizeSSL.java Normal file
View File

@ -0,0 +1,174 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.util;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
/**
* Marker annotation indicating when SSL Randomization should be used for a test class, and if so what
* the typical odds of using SSL should for that test class.
* @see SSLRandomizer#getSSLRandomizerForClass
* @see SuppressSSL
*/
@Documented
@Inherited
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
public @interface RandomizeSSL {
// we don't choose ssl that often by default because of SOLR-5776
public static final double DEFAULT_ODDS = 0.2D;
/** Comment to inlcude when logging details of SSL randomization */
public String reason() default "";
/**
* Odds (as ratio relative to 1) that SSL should be selected in a typical run.
* Must either be betwen 0.0 and 1.0 (inclusively) or NaN in which case a sensible should be used.
* Actual Odds used for randomization may be higher depending on runner options such as
* <code>tests.multiplier</code> or <code>tests.nightly</code>
*
* @see #DEFAULT_ODDS
* @see LuceneTestCase#TEST_NIGHTLY
* @see LuceneTestCase#RANDOM_MULTIPLIER
*/
public double ssl() default Double.NaN;
/**
* Odds (as ratio relative to 1) that SSL should be selected in a typical run.
* Must either be betwen 0.0 and 1.0 (inclusively) or NaN in which case the effective value of
* {@link #ssl} should be used.
* Actual Odds used for randomization may be higher depending on runner options such as
* <code>tests.multiplier</code> or <code>tests.nightly</code>
* <p>
* NOTE: clientAuth is useless unless ssl is also in used, but we randomize it independently
* just in case it might find bugs in our test/ssl client code (ie: attempting to use
* SSL w/client cert to non-ssl servers)
* </p>
* @see #DEFAULT_ODDS
* @see LuceneTestCase#TEST_NIGHTLY
* @see LuceneTestCase#RANDOM_MULTIPLIER
*/
public double clientAuth() default Double.NaN;
/**
* A shorthand option for controlling both {@link #ssl} and {@link #clientAuth} with a single numeric
* value, For example: <code>@RandomizeSSL(0.5)</code>.
*
* Ignored if {@link #ssl} is set explicitly.
*/
public double value() default Double.NaN;
/**
* A simple data structure for encapsulating the effective values to be used when randomizing
* SSL in a test, based on the configured values in the {@link RandomizeSSL} annotation.
*/
public static final class SSLRandomizer {
public final double ssl;
public final double clientAuth;
public final String debug;
/** @lucene.internal */
public SSLRandomizer(double ssl, double clientAuth, String debug) {
this.ssl = ssl;
this.clientAuth = clientAuth;
this.debug = debug;
}
/**
* Randomly produces an SSLTestConfig taking into account various factors
*
* @see LuceneTestCase#TEST_NIGHTLY
* @see LuceneTestCase#RANDOM_MULTIPLIER
* @see LuceneTestCase#random()
*/
public SSLTestConfig createSSLTestConfig() {
// even if we know SSL is disabled, always consume the same amount of randomness
// that way all other test behavior should be consistent even if a user adds/removes @SuppressSSL
final boolean useSSL = TestUtil.nextInt(LuceneTestCase.random(), 0, 1000) <
(int)(1000 * getEffectiveOdds(ssl, LuceneTestCase.TEST_NIGHTLY, LuceneTestCase.RANDOM_MULTIPLIER));
final boolean useClientAuth = TestUtil.nextInt(LuceneTestCase.random(), 0, 1000) <
(int)(1000 * getEffectiveOdds(clientAuth, LuceneTestCase.TEST_NIGHTLY, LuceneTestCase.RANDOM_MULTIPLIER));
return new SSLTestConfig(useSSL, useClientAuth);
}
/** @lucene.internal Public only for testing */
public static double getEffectiveOdds(final double declaredOdds,
final boolean nightly,
final int multiplier) {
assert declaredOdds <= 1.0D;
assert 0.0D <= declaredOdds;
if (declaredOdds == 0.0D || declaredOdds == 1.0D ) {
return declaredOdds;
}
assert 0 < multiplier;
// negate the odds so we can then divide it by our multipling factors
// to increase the final odds
return 1.0D - ((1.0D - declaredOdds)
/ ((nightly ? 1.1D : 1.0D) * (1.0D + Math.log(multiplier))));
}
/**
* Returns an SSLRandomizer suitable for the specified (test) class
*/
public static final SSLRandomizer getSSLRandomizerForClass(Class clazz) {
final SuppressSSL suppression = (SuppressSSL) clazz.getAnnotation(SuppressSSL.class);
if (null != suppression) {
// Even if this class has a RandomizeSSL annotation, any usage of SuppressSSL -- even in a
// super class -- overrules that.
//
// (If it didn't work this way, it would be a pain in the ass to quickly disable SSL for a
// broad hierarchy of tests)
return new SSLRandomizer(0.0D, 0.0D, suppression.toString());
}
final RandomizeSSL annotation = (RandomizeSSL) clazz.getAnnotation(RandomizeSSL.class);
if (null == annotation) {
return new SSLRandomizer(0.0D, 0.0D, RandomizeSSL.class.getName() + " annotation not specified");
}
final double def = Double.isNaN(annotation.value()) ? DEFAULT_ODDS : annotation.value();
if (def < 0.0D || 1.0D < def) {
throw new IllegalArgumentException
(clazz.getName() + ": default value is not a ratio between 0 and 1: " + annotation.toString());
}
final double ssl = Double.isNaN(annotation.ssl()) ? def : annotation.ssl();
if (ssl < 0.0D || 1.0D < ssl) {
throw new IllegalArgumentException
(clazz.getName() + ": ssl value is not a ratio between 0 and 1: " + annotation.toString());
}
final double clientAuth = Double.isNaN(annotation.clientAuth()) ? ssl : annotation.clientAuth();
if (clientAuth < 0.0D || 1 < clientAuth) {
throw new IllegalArgumentException
(clazz.getName() + ": clientAuth value is not a ratio between 0 and 1: " + annotation.toString());
}
return new SSLRandomizer(ssl, clientAuth, annotation.toString());
}
}
}