mirror of https://github.com/apache/lucene.git
Merge branch 'master' into shared_min_score
This commit is contained in:
jimczi
commit
ee50a3812c
|
|
@ -55,6 +55,9 @@ Improvements
|
|||
|
||||
* LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via Anshum Gupta)
|
||||
|
||||
* LUCENE-8213: LRUQueryCache#doCache now uses IndexSearcher's Executor (if present)
|
||||
to asynchronously cache heavy queries (Atri Sharma)
|
||||
|
||||
Bug fixes
|
||||
|
||||
* LUCENE-8663: NRTCachingDirectory.slowFileExists may open a file while
|
||||
|
|
|
|||
|
|
@ -181,7 +181,16 @@ public class IndexSearcher {
|
|||
}
|
||||
|
||||
/** Runs searches for each segment separately, using the
|
||||
* provided Executor. NOTE:
|
||||
* provided Executor. The passed in Executor will also be
|
||||
* used by LRUQueryCache (if enabled) to perform asynchronous
|
||||
* query caching.
|
||||
* If a task is rejected by the host Executor, the failed task
|
||||
* will then be executed on the caller thread. This is done to
|
||||
* ensure that a query succeeds, albeit with a higher latency.
|
||||
* If a user wishes to modify the said behaviour, they can either
|
||||
* handle the exception in the provided Executor, or override
|
||||
* the said method in a custom extension of IndexSearcher.
|
||||
* NOTE:
|
||||
* if you are using {@link NIOFSDirectory}, do not use
|
||||
* the shutdownNow method of ExecutorService as this uses
|
||||
* Thread.interrupt under-the-hood which can silently
|
||||
|
|
@ -842,7 +851,11 @@ public class IndexSearcher {
|
|||
final QueryCache queryCache = this.queryCache;
|
||||
Weight weight = query.createWeight(this, scoreMode, boost);
|
||||
if (scoreMode.needsScores() == false && queryCache != null) {
|
||||
weight = queryCache.doCache(weight, queryCachingPolicy);
|
||||
if (executor != null) {
|
||||
weight = queryCache.doCache(weight, queryCachingPolicy, executor);
|
||||
} else {
|
||||
weight = queryCache.doCache(weight, queryCachingPolicy);
|
||||
}
|
||||
}
|
||||
return weight;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,12 +22,16 @@ import java.util.ArrayList;
|
|||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.ConcurrentModificationException;
|
||||
import java.util.HashSet;
|
||||
import java.util.IdentityHashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.FutureTask;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.function.Predicate;
|
||||
|
|
@ -95,6 +99,11 @@ public class LRUQueryCache implements QueryCache, Accountable {
|
|||
// maps queries that are contained in the cache to a singleton so that this
|
||||
// cache does not store several copies of the same query
|
||||
private final Map<Query, Query> uniqueQueries;
|
||||
// Marks the inflight queries that are being asynchronously loaded into the cache
|
||||
// This is used to ensure that multiple threads do not trigger loading
|
||||
// of the same query in the same cache. We use a set because it is an invariant that
|
||||
// the entries of this data structure be unique.
|
||||
private final Set<Query> inFlightAsyncLoadQueries = new HashSet<>();
|
||||
// The contract between this set and the per-leaf caches is that per-leaf caches
|
||||
// are only allowed to store sub-sets of the queries that are contained in
|
||||
// mostRecentlyUsedQueries. This is why write operations are performed under a lock
|
||||
|
|
@ -262,6 +271,7 @@ public class LRUQueryCache implements QueryCache, Accountable {
|
|||
assert lock.isHeldByCurrentThread();
|
||||
assert key instanceof BoostQuery == false;
|
||||
assert key instanceof ConstantScoreQuery == false;
|
||||
|
||||
final IndexReader.CacheKey readerKey = cacheHelper.getKey();
|
||||
final LeafCache leafCache = cache.get(readerKey);
|
||||
if (leafCache == null) {
|
||||
|
|
@ -368,6 +378,7 @@ public class LRUQueryCache implements QueryCache, Accountable {
|
|||
onEviction(singleton);
|
||||
}
|
||||
} finally {
|
||||
inFlightAsyncLoadQueries.remove(query);
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
|
@ -389,6 +400,7 @@ public class LRUQueryCache implements QueryCache, Accountable {
|
|||
cache.clear();
|
||||
// Note that this also clears the uniqueQueries map since mostRecentlyUsedQueries is the uniqueQueries.keySet view:
|
||||
mostRecentlyUsedQueries.clear();
|
||||
inFlightAsyncLoadQueries.clear();
|
||||
onClear();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
|
|
@ -448,13 +460,37 @@ public class LRUQueryCache implements QueryCache, Accountable {
|
|||
}
|
||||
}
|
||||
|
||||
// pkg-private for testing
|
||||
// return the list of queries being loaded asynchronously
|
||||
List<Query> inFlightQueries() {
|
||||
return new ArrayList<>(inFlightAsyncLoadQueries);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Weight doCache(Weight weight, QueryCachingPolicy policy) {
|
||||
Weight originalWeight = getOriginalWeight(weight);
|
||||
|
||||
return new CachingWrapperWeight(originalWeight, policy);
|
||||
}
|
||||
|
||||
// Should be used only when the user wishes to trade throughput for latency
|
||||
// This method was not merged in the method above as to not break the existing contract
|
||||
// advertised by QueryCache
|
||||
@Override
|
||||
public Weight doCache(final Weight weight, QueryCachingPolicy policy, Executor executor) {
|
||||
assert executor != null;
|
||||
Weight originalWeight = getOriginalWeight(weight);
|
||||
|
||||
return new CachingWrapperWeight(originalWeight, policy, executor);
|
||||
}
|
||||
|
||||
// Get original weight from the cached weight
|
||||
private Weight getOriginalWeight(Weight weight) {
|
||||
while (weight instanceof CachingWrapperWeight) {
|
||||
weight = ((CachingWrapperWeight) weight).in;
|
||||
}
|
||||
|
||||
return new CachingWrapperWeight(weight, policy);
|
||||
return weight;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -656,10 +692,21 @@ public class LRUQueryCache implements QueryCache, Accountable {
|
|||
// threads when IndexSearcher is created with threads
|
||||
private final AtomicBoolean used;
|
||||
|
||||
private final Executor executor;
|
||||
|
||||
CachingWrapperWeight(Weight in, QueryCachingPolicy policy) {
|
||||
super(in.getQuery(), 1f);
|
||||
this.in = in;
|
||||
this.policy = policy;
|
||||
this.executor = null;
|
||||
used = new AtomicBoolean(false);
|
||||
}
|
||||
|
||||
CachingWrapperWeight(Weight in, QueryCachingPolicy policy, Executor executor) {
|
||||
super(in.getQuery(), 1f);
|
||||
this.in = in;
|
||||
this.policy = policy;
|
||||
this.executor = executor;
|
||||
used = new AtomicBoolean(false);
|
||||
}
|
||||
|
||||
|
|
@ -732,8 +779,24 @@ public class LRUQueryCache implements QueryCache, Accountable {
|
|||
|
||||
if (docIdSet == null) {
|
||||
if (policy.shouldCache(in.getQuery())) {
|
||||
docIdSet = cache(context);
|
||||
putIfAbsent(in.getQuery(), docIdSet, cacheHelper);
|
||||
boolean cacheSynchronously = executor == null;
|
||||
|
||||
// If asynchronous caching is requested, perform the same and return
|
||||
// the uncached iterator
|
||||
if (cacheSynchronously == false) {
|
||||
cacheSynchronously = cacheAsynchronously(context, cacheHelper);
|
||||
|
||||
// If async caching failed, synchronous caching will
|
||||
// be performed, hence do not return the uncached value
|
||||
if (cacheSynchronously == false) {
|
||||
return in.scorerSupplier(context);
|
||||
}
|
||||
}
|
||||
|
||||
if (cacheSynchronously) {
|
||||
docIdSet = cache(context);
|
||||
putIfAbsent(in.getQuery(), docIdSet, cacheHelper);
|
||||
}
|
||||
} else {
|
||||
return in.scorerSupplier(context);
|
||||
}
|
||||
|
|
@ -813,8 +876,23 @@ public class LRUQueryCache implements QueryCache, Accountable {
|
|||
|
||||
if (docIdSet == null) {
|
||||
if (policy.shouldCache(in.getQuery())) {
|
||||
docIdSet = cache(context);
|
||||
putIfAbsent(in.getQuery(), docIdSet, cacheHelper);
|
||||
boolean cacheSynchronously = executor == null;
|
||||
// If asynchronous caching is requested, perform the same and return
|
||||
// the uncached iterator
|
||||
if (cacheSynchronously == false) {
|
||||
cacheSynchronously = cacheAsynchronously(context, cacheHelper);
|
||||
|
||||
// If async caching failed, we will perform synchronous caching
|
||||
// hence do not return the uncached value here
|
||||
if (cacheSynchronously == false) {
|
||||
return in.bulkScorer(context);
|
||||
}
|
||||
}
|
||||
|
||||
if (cacheSynchronously) {
|
||||
docIdSet = cache(context);
|
||||
putIfAbsent(in.getQuery(), docIdSet, cacheHelper);
|
||||
}
|
||||
} else {
|
||||
return in.bulkScorer(context);
|
||||
}
|
||||
|
|
@ -832,5 +910,38 @@ public class LRUQueryCache implements QueryCache, Accountable {
|
|||
return new DefaultBulkScorer(new ConstantScoreScorer(this, 0f, ScoreMode.COMPLETE_NO_SCORES, disi));
|
||||
}
|
||||
|
||||
// Perform a cache load asynchronously
|
||||
// @return true if synchronous caching is needed, false otherwise
|
||||
private boolean cacheAsynchronously(LeafReaderContext context, IndexReader.CacheHelper cacheHelper) {
|
||||
/*
|
||||
* If the current query is already being asynchronously cached,
|
||||
* do not trigger another cache operation
|
||||
*/
|
||||
if (inFlightAsyncLoadQueries.add(in.getQuery()) == false) {
|
||||
return false;
|
||||
}
|
||||
|
||||
FutureTask<Void> task = new FutureTask<>(() -> {
|
||||
DocIdSet localDocIdSet = cache(context);
|
||||
putIfAbsent(in.getQuery(), localDocIdSet, cacheHelper);
|
||||
|
||||
// Remove the key from inflight -- the key is loaded now
|
||||
Object retValue = inFlightAsyncLoadQueries.remove(in.getQuery());
|
||||
|
||||
// The query should have been present in the inflight queries set before
|
||||
// we actually loaded it -- hence the removal of the key should be successful
|
||||
assert retValue != null;
|
||||
|
||||
return null;
|
||||
});
|
||||
try {
|
||||
executor.execute(task);
|
||||
} catch (RejectedExecutionException e) {
|
||||
// Trigger synchronous caching
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@
|
|||
package org.apache.lucene.search;
|
||||
|
||||
|
||||
import java.util.concurrent.Executor;
|
||||
|
||||
/**
|
||||
* A cache for queries.
|
||||
*
|
||||
|
|
@ -33,4 +35,10 @@ public interface QueryCache {
|
|||
*/
|
||||
Weight doCache(Weight weight, QueryCachingPolicy policy);
|
||||
|
||||
/**
|
||||
* Same as above, but allows passing in an Executor to perform caching
|
||||
* asynchronously
|
||||
*/
|
||||
Weight doCache(Weight weight, QueryCachingPolicy policy, Executor executor);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import java.util.Collection;
|
|||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Executor;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
|
@ -189,6 +190,11 @@ public class TestIndexSearcher extends LuceneTestCase {
|
|||
QueryCache dummyCache = new QueryCache() {
|
||||
@Override
|
||||
public Weight doCache(Weight weight, QueryCachingPolicy policy) {
|
||||
return doCache(weight, policy, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Weight doCache(Weight weight, QueryCachingPolicy policy, Executor executor) {
|
||||
return weight;
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -30,6 +30,11 @@ import java.util.LinkedHashMap;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.FutureTask;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
|
@ -59,6 +64,7 @@ import org.apache.lucene.store.Directory;
|
|||
import org.apache.lucene.util.Constants;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.lucene.util.NamedThreadFactory;
|
||||
import org.apache.lucene.util.RamUsageTester;
|
||||
import org.apache.lucene.util.TestUtil;
|
||||
|
||||
|
|
@ -96,6 +102,9 @@ public class TestLRUQueryCache extends LuceneTestCase {
|
|||
final LRUQueryCache queryCache = new LRUQueryCache(1 + random().nextInt(20), 1 + random().nextInt(10000), context -> random().nextBoolean());
|
||||
Directory dir = newDirectory();
|
||||
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
|
||||
ExecutorService service = new ThreadPoolExecutor(4, 4, 0L, TimeUnit.MILLISECONDS,
|
||||
new LinkedBlockingQueue<Runnable>(),
|
||||
new NamedThreadFactory("TestLRUQueryCache"));
|
||||
final SearcherFactory searcherFactory = new SearcherFactory() {
|
||||
@Override
|
||||
public IndexSearcher newSearcher(IndexReader reader, IndexReader previous) throws IOException {
|
||||
|
|
@ -105,87 +114,103 @@ public class TestLRUQueryCache extends LuceneTestCase {
|
|||
return searcher;
|
||||
}
|
||||
};
|
||||
final boolean applyDeletes = random().nextBoolean();
|
||||
final SearcherManager mgr = new SearcherManager(w.w, applyDeletes, false, searcherFactory);
|
||||
final AtomicBoolean indexing = new AtomicBoolean(true);
|
||||
final AtomicReference<Throwable> error = new AtomicReference<>();
|
||||
final int numDocs = atLeast(10000);
|
||||
Thread[] threads = new Thread[3];
|
||||
threads[0] = new Thread() {
|
||||
public void run() {
|
||||
Document doc = new Document();
|
||||
StringField f = new StringField("color", "", Store.NO);
|
||||
doc.add(f);
|
||||
for (int i = 0; indexing.get() && i < numDocs; ++i) {
|
||||
f.setStringValue(RandomPicks.randomFrom(random(), new String[] {"blue", "red", "yellow"}));
|
||||
try {
|
||||
w.addDocument(doc);
|
||||
if ((i & 63) == 0) {
|
||||
mgr.maybeRefresh();
|
||||
if (rarely()) {
|
||||
queryCache.clear();
|
||||
}
|
||||
if (rarely()) {
|
||||
final String color = RandomPicks.randomFrom(random(), new String[] {"blue", "red", "yellow"});
|
||||
w.deleteDocuments(new Term("color", color));
|
||||
}
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
error.compareAndSet(null, t);
|
||||
break;
|
||||
}
|
||||
}
|
||||
indexing.set(false);
|
||||
final SearcherFactory concurrentSearcherFactory = new SearcherFactory() {
|
||||
@Override
|
||||
public IndexSearcher newSearcher(IndexReader reader, IndexReader previous) throws IOException {
|
||||
IndexSearcher searcher = new IndexSearcher(reader, service);
|
||||
searcher.setQueryCachingPolicy(MAYBE_CACHE_POLICY);
|
||||
searcher.setQueryCache(queryCache);
|
||||
return searcher;
|
||||
}
|
||||
};
|
||||
for (int i = 1; i < threads.length; ++i) {
|
||||
threads[i] = new Thread() {
|
||||
@Override
|
||||
|
||||
final SearcherFactory[] searcherFactories = {searcherFactory, concurrentSearcherFactory};
|
||||
|
||||
for (SearcherFactory currentSearcherFactory : searcherFactories) {
|
||||
final boolean applyDeletes = random().nextBoolean();
|
||||
final SearcherManager mgr = new SearcherManager(w.w, applyDeletes, false, currentSearcherFactory);
|
||||
final AtomicBoolean indexing = new AtomicBoolean(true);
|
||||
final AtomicReference<Throwable> error = new AtomicReference<>();
|
||||
final int numDocs = atLeast(10000);
|
||||
Thread[] threads = new Thread[3];
|
||||
threads[0] = new Thread() {
|
||||
public void run() {
|
||||
while (indexing.get()) {
|
||||
Document doc = new Document();
|
||||
StringField f = new StringField("color", "", Store.NO);
|
||||
doc.add(f);
|
||||
for (int i = 0; indexing.get() && i < numDocs; ++i) {
|
||||
f.setStringValue(RandomPicks.randomFrom(random(), new String[]{"blue", "red", "yellow"}));
|
||||
try {
|
||||
final IndexSearcher searcher = mgr.acquire();
|
||||
try {
|
||||
final String value = RandomPicks.randomFrom(random(), new String[] {"blue", "red", "yellow", "green"});
|
||||
final Query q = new TermQuery(new Term("color", value));
|
||||
TotalHitCountCollector collector = new TotalHitCountCollector();
|
||||
searcher.search(q, collector); // will use the cache
|
||||
final int totalHits1 = collector.getTotalHits();
|
||||
TotalHitCountCollector collector2 = new TotalHitCountCollector();
|
||||
searcher.search(q, new FilterCollector(collector2) {
|
||||
public ScoreMode scoreMode() {
|
||||
return ScoreMode.COMPLETE; // will not use the cache because of scores
|
||||
}
|
||||
});
|
||||
final long totalHits2 = collector2.getTotalHits();
|
||||
assertEquals(totalHits2, totalHits1);
|
||||
} finally {
|
||||
mgr.release(searcher);
|
||||
w.addDocument(doc);
|
||||
if ((i & 63) == 0) {
|
||||
mgr.maybeRefresh();
|
||||
if (rarely()) {
|
||||
queryCache.clear();
|
||||
}
|
||||
if (rarely()) {
|
||||
final String color = RandomPicks.randomFrom(random(), new String[]{"blue", "red", "yellow"});
|
||||
w.deleteDocuments(new Term("color", color));
|
||||
}
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
error.compareAndSet(null, t);
|
||||
break;
|
||||
}
|
||||
}
|
||||
indexing.set(false);
|
||||
}
|
||||
};
|
||||
for (int i = 1; i < threads.length; ++i) {
|
||||
threads[i] = new Thread() {
|
||||
@Override
|
||||
public void run() {
|
||||
while (indexing.get()) {
|
||||
try {
|
||||
final IndexSearcher searcher = mgr.acquire();
|
||||
try {
|
||||
final String value = RandomPicks.randomFrom(random(), new String[]{"blue", "red", "yellow", "green"});
|
||||
final Query q = new TermQuery(new Term("color", value));
|
||||
TotalHitCountCollector collector = new TotalHitCountCollector();
|
||||
searcher.search(q, collector); // will use the cache
|
||||
final int totalHits1 = collector.getTotalHits();
|
||||
TotalHitCountCollector collector2 = new TotalHitCountCollector();
|
||||
searcher.search(q, new FilterCollector(collector2) {
|
||||
public ScoreMode scoreMode() {
|
||||
return ScoreMode.COMPLETE; // will not use the cache because of scores
|
||||
}
|
||||
});
|
||||
final long totalHits2 = collector2.getTotalHits();
|
||||
assertEquals(totalHits2, totalHits1);
|
||||
} finally {
|
||||
mgr.release(searcher);
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
error.compareAndSet(null, t);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
for (Thread thread : threads) {
|
||||
thread.start();
|
||||
}
|
||||
|
||||
for (Thread thread : threads) {
|
||||
thread.join();
|
||||
}
|
||||
|
||||
if (error.get() != null) {
|
||||
throw error.get();
|
||||
}
|
||||
queryCache.assertConsistent();
|
||||
mgr.close();
|
||||
}
|
||||
|
||||
for (Thread thread : threads) {
|
||||
thread.start();
|
||||
}
|
||||
|
||||
for (Thread thread : threads) {
|
||||
thread.join();
|
||||
}
|
||||
|
||||
if (error.get() != null) {
|
||||
throw error.get();
|
||||
}
|
||||
queryCache.assertConsistent();
|
||||
mgr.close();
|
||||
w.close();
|
||||
dir.close();
|
||||
queryCache.assertConsistent();
|
||||
service.shutdown();
|
||||
}
|
||||
|
||||
public void testLRUEviction() throws Exception {
|
||||
|
|
@ -201,7 +226,9 @@ public class TestLRUQueryCache extends LuceneTestCase {
|
|||
f.setStringValue("green");
|
||||
w.addDocument(doc);
|
||||
final DirectoryReader reader = w.getReader();
|
||||
final IndexSearcher searcher = newSearcher(reader);
|
||||
|
||||
IndexSearcher searcher = newSearcher(reader);
|
||||
|
||||
final LRUQueryCache queryCache = new LRUQueryCache(2, 100000, context -> true);
|
||||
|
||||
final Query blue = new TermQuery(new Term("color", "blue"));
|
||||
|
|
@ -218,22 +245,50 @@ public class TestLRUQueryCache extends LuceneTestCase {
|
|||
|
||||
searcher.setQueryCachingPolicy(ALWAYS_CACHE);
|
||||
searcher.search(new ConstantScoreQuery(red), 1);
|
||||
assertEquals(Collections.singletonList(red), queryCache.cachedQueries());
|
||||
|
||||
if (!(queryCache.cachedQueries().equals(Collections.emptyList()))) {
|
||||
assertEquals(Arrays.asList(red), queryCache.cachedQueries());
|
||||
} else {
|
||||
// Let the cache load be completed
|
||||
Thread.sleep(200);
|
||||
assertEquals(Arrays.asList(red), queryCache.cachedQueries());
|
||||
}
|
||||
|
||||
searcher.search(new ConstantScoreQuery(green), 1);
|
||||
assertEquals(Arrays.asList(red, green), queryCache.cachedQueries());
|
||||
|
||||
if (!(queryCache.cachedQueries().equals(Arrays.asList(red)))) {
|
||||
assertEquals(Arrays.asList(red, green), queryCache.cachedQueries());
|
||||
} else {
|
||||
// Let the cache load be completed
|
||||
Thread.sleep(200);
|
||||
assertEquals(Arrays.asList(red, green), queryCache.cachedQueries());
|
||||
}
|
||||
|
||||
searcher.search(new ConstantScoreQuery(red), 1);
|
||||
assertEquals(Arrays.asList(green, red), queryCache.cachedQueries());
|
||||
|
||||
searcher.search(new ConstantScoreQuery(blue), 1);
|
||||
assertEquals(Arrays.asList(red, blue), queryCache.cachedQueries());
|
||||
|
||||
if (!(queryCache.cachedQueries().equals(Arrays.asList(green, red)))) {
|
||||
assertEquals(Arrays.asList(red, blue), queryCache.cachedQueries());
|
||||
} else {
|
||||
// Let the cache load be completed
|
||||
Thread.sleep(200);
|
||||
assertEquals(Arrays.asList(red, blue), queryCache.cachedQueries());
|
||||
}
|
||||
|
||||
searcher.search(new ConstantScoreQuery(blue), 1);
|
||||
assertEquals(Arrays.asList(red, blue), queryCache.cachedQueries());
|
||||
|
||||
searcher.search(new ConstantScoreQuery(green), 1);
|
||||
assertEquals(Arrays.asList(blue, green), queryCache.cachedQueries());
|
||||
|
||||
if (!(queryCache.cachedQueries().equals(Arrays.asList(red, blue)))) {
|
||||
assertEquals(Arrays.asList(blue, green), queryCache.cachedQueries());
|
||||
} else {
|
||||
// Let the cache load be completed
|
||||
Thread.sleep(200);
|
||||
assertEquals(Arrays.asList(blue, green), queryCache.cachedQueries());
|
||||
}
|
||||
|
||||
searcher.setQueryCachingPolicy(NEVER_CACHE);
|
||||
searcher.search(new ConstantScoreQuery(red), 1);
|
||||
|
|
@ -244,6 +299,150 @@ public class TestLRUQueryCache extends LuceneTestCase {
|
|||
dir.close();
|
||||
}
|
||||
|
||||
public void testLRUConcurrentLoadAndEviction() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
|
||||
|
||||
Document doc = new Document();
|
||||
StringField f = new StringField("color", "blue", Store.NO);
|
||||
doc.add(f);
|
||||
w.addDocument(doc);
|
||||
f.setStringValue("red");
|
||||
w.addDocument(doc);
|
||||
f.setStringValue("green");
|
||||
w.addDocument(doc);
|
||||
final DirectoryReader reader = w.getReader();
|
||||
ExecutorService service = new ThreadPoolExecutor(4, 4, 0L, TimeUnit.MILLISECONDS,
|
||||
new LinkedBlockingQueue<Runnable>(),
|
||||
new NamedThreadFactory("TestLRUQueryCache"));
|
||||
|
||||
IndexSearcher searcher = new IndexSearcher(reader, service);
|
||||
|
||||
final LRUQueryCache queryCache = new LRUQueryCache(2, 100000, context -> true);
|
||||
|
||||
final Query blue = new TermQuery(new Term("color", "blue"));
|
||||
final Query red = new TermQuery(new Term("color", "red"));
|
||||
final Query green = new TermQuery(new Term("color", "green"));
|
||||
|
||||
assertEquals(Collections.emptyList(), queryCache.cachedQueries());
|
||||
|
||||
searcher.setQueryCache(queryCache);
|
||||
// the filter is not cached on any segment: no changes
|
||||
searcher.setQueryCachingPolicy(NEVER_CACHE);
|
||||
searcher.search(new ConstantScoreQuery(green), 1);
|
||||
assertEquals(Collections.emptyList(), queryCache.cachedQueries());
|
||||
|
||||
searcher.setQueryCachingPolicy(ALWAYS_CACHE);
|
||||
// First read should miss
|
||||
searcher.search(new ConstantScoreQuery(red), 1);
|
||||
|
||||
if (!(queryCache.cachedQueries().equals(Collections.emptyList()))) {
|
||||
searcher.search(new ConstantScoreQuery(red), 1);
|
||||
} else {
|
||||
// Let the cache load be completed
|
||||
Thread.sleep(200);
|
||||
searcher.search(new ConstantScoreQuery(red), 1);
|
||||
}
|
||||
|
||||
// Second read should hit
|
||||
searcher.search(new ConstantScoreQuery(red), 1);
|
||||
assertEquals(Collections.singletonList(red), queryCache.cachedQueries());
|
||||
|
||||
searcher.search(new ConstantScoreQuery(green), 1);
|
||||
if (!(queryCache.cachedQueries().equals(Arrays.asList(red)))) {
|
||||
assertEquals(Arrays.asList(red, green), queryCache.cachedQueries());
|
||||
} else {
|
||||
// Let the cache load be completed
|
||||
Thread.sleep(200);
|
||||
assertEquals(Arrays.asList(red, green), queryCache.cachedQueries());
|
||||
}
|
||||
|
||||
searcher.search(new ConstantScoreQuery(red), 1);
|
||||
assertEquals(Arrays.asList(green, red), queryCache.cachedQueries());
|
||||
|
||||
searcher.search(new ConstantScoreQuery(blue), 1);
|
||||
if (!(queryCache.cachedQueries().equals(Arrays.asList(green, red)))) {
|
||||
assertEquals(Arrays.asList(red, blue), queryCache.cachedQueries());
|
||||
} else {
|
||||
// Let the cache load be completed
|
||||
Thread.sleep(200);
|
||||
assertEquals(Arrays.asList(red, blue), queryCache.cachedQueries());
|
||||
}
|
||||
|
||||
searcher.search(new ConstantScoreQuery(blue), 1);
|
||||
assertEquals(Arrays.asList(red, blue), queryCache.cachedQueries());
|
||||
|
||||
searcher.search(new ConstantScoreQuery(green), 1);
|
||||
if (!(queryCache.cachedQueries().equals(Arrays.asList(red, blue)))) {
|
||||
assertEquals(Arrays.asList(blue, green), queryCache.cachedQueries());
|
||||
} else {
|
||||
// Let the cache load be completed
|
||||
Thread.sleep(200);
|
||||
assertEquals(Arrays.asList(blue, green), queryCache.cachedQueries());
|
||||
}
|
||||
|
||||
searcher.setQueryCachingPolicy(NEVER_CACHE);
|
||||
searcher.search(new ConstantScoreQuery(red), 1);
|
||||
assertEquals(Arrays.asList(blue, green), queryCache.cachedQueries());
|
||||
|
||||
reader.close();
|
||||
w.close();
|
||||
dir.close();
|
||||
service.shutdown();
|
||||
}
|
||||
|
||||
public void testLRUConcurrentLoadsOfSameQuery() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
|
||||
|
||||
Document doc = new Document();
|
||||
StringField f = new StringField("color", "blue", Store.NO);
|
||||
doc.add(f);
|
||||
w.addDocument(doc);
|
||||
f.setStringValue("red");
|
||||
w.addDocument(doc);
|
||||
f.setStringValue("green");
|
||||
w.addDocument(doc);
|
||||
final DirectoryReader reader = w.getReader();
|
||||
ExecutorService service = new ThreadPoolExecutor(4, 4, 0L, TimeUnit.MILLISECONDS,
|
||||
new LinkedBlockingQueue<Runnable>(),
|
||||
new NamedThreadFactory("TestLRUQueryCache"));
|
||||
|
||||
ExecutorService stressService = new ThreadPoolExecutor(15, 15, 0L, TimeUnit.MILLISECONDS,
|
||||
new LinkedBlockingQueue<Runnable>(),
|
||||
new NamedThreadFactory("TestLRUQueryCache2"));
|
||||
|
||||
IndexSearcher searcher = new IndexSearcher(reader, service);
|
||||
|
||||
final LRUQueryCache queryCache = new LRUQueryCache(2, 100000, context -> true);
|
||||
|
||||
final Query green = new TermQuery(new Term("color", "green"));
|
||||
|
||||
assertEquals(Collections.emptyList(), queryCache.cachedQueries());
|
||||
|
||||
searcher.setQueryCache(queryCache);
|
||||
searcher.setQueryCachingPolicy(ALWAYS_CACHE);
|
||||
|
||||
FutureTask<Void> task = new FutureTask<>(() -> {
|
||||
searcher.search(new ConstantScoreQuery(green), 1);
|
||||
assertEquals(1, queryCache.inFlightQueries().size());
|
||||
return null;
|
||||
});
|
||||
|
||||
for (int i = 0; i < 5; i++) {
|
||||
stressService.submit(task);
|
||||
}
|
||||
|
||||
Thread.sleep(3000);
|
||||
assertEquals(Arrays.asList(green), queryCache.cachedQueries());
|
||||
|
||||
reader.close();
|
||||
w.close();
|
||||
dir.close();
|
||||
service.shutdown();
|
||||
stressService.shutdown();
|
||||
}
|
||||
|
||||
public void testClearFilter() throws IOException {
|
||||
Directory dir = newDirectory();
|
||||
final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
|
||||
|
|
|
|||
|
|
@ -144,6 +144,12 @@ New Features
|
|||
* SOLR-13272: Add support for arbitrary ranges in JSON facet's Range facets.
|
||||
(Apoorv Bhawsar, Munendra S N, Mikhail Khludnev, Ishan Chattopadhyaya, Jan Høydahl)
|
||||
|
||||
* SOLR-13632: Support integral plots, cosine distance and string truncation with math expressions (Joel Bernstein)
|
||||
|
||||
* SOLR-13667: Add upper, lower, trim and split Stream Evaluators (Joel Bernstein)
|
||||
|
||||
* SOLR-13625: Add CsvStream, TsvStream Streaming Expressions and supporting Stream Evaluators (Joel bernstein)
|
||||
|
||||
Improvements
|
||||
----------------------
|
||||
|
||||
|
|
@ -238,6 +244,17 @@ Bug Fixes
|
|||
|
||||
* SOLR-13022: Fix NPE when sorting by non-existent aggregate function in JSON Facet (hossman, Munendra S N)
|
||||
|
||||
* SOLR-13727: Fixed V2Requests - HttpSolrClient replaced first instance of "/solr" with "/api" which
|
||||
caused a change in host names starting with "solr". (Megan Carey via yonik)
|
||||
|
||||
* SOLR-13180: Fix ClassCastException in Json Request API (Johannes Kloos, Jan Høydahl, Munendra S N)
|
||||
|
||||
* SOLR-13417: Handle stats aggregation on date and string fields in SolrJ's JSON facet response processing
|
||||
(Jason Gerlowski, Munendra S N)
|
||||
|
||||
* SOLR-13712: JMX MBeans are not exposed because of race condition between creating platform mbean server and
|
||||
registering mbeans. (shalin)
|
||||
|
||||
Other Changes
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ public class SolrJmxReporter extends FilteringSolrMetricReporter {
|
|||
protected synchronized void doInit() {
|
||||
if (serviceUrl != null && agentId != null) {
|
||||
mBeanServer = JmxUtil.findFirstMBeanServer();
|
||||
log.warn("No more than one of serviceUrl({}) and agentId({}) should be configured, using first MBeanServer instead of configuration.",
|
||||
log.warn("No more than one of serviceUrl({}) and agentId({}) should be configured, using first MBeanServer {} instead of configuration.",
|
||||
serviceUrl, agentId, mBeanServer);
|
||||
} else if (serviceUrl != null) {
|
||||
// reuse existing services
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import javax.management.Query;
|
|||
import javax.management.QueryExp;
|
||||
import java.io.Closeable;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.util.HashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
|
@ -157,9 +156,6 @@ public class JmxMetricsReporter implements Reporter, Closeable {
|
|||
}
|
||||
|
||||
public JmxMetricsReporter build() {
|
||||
if (mBeanServer == null) {
|
||||
mBeanServer = ManagementFactory.getPlatformMBeanServer();
|
||||
}
|
||||
if (tag == null) {
|
||||
tag = Integer.toHexString(this.hashCode());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ import java.util.LinkedHashMap;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
||||
public class ObjectUtil {
|
||||
|
||||
public static class ConflictHandler {
|
||||
|
|
@ -103,10 +105,14 @@ public class ObjectUtil {
|
|||
// OK, now we need to merge values
|
||||
handler.handleConflict(outer, path, key, val, existingVal);
|
||||
}
|
||||
} else {
|
||||
} else if (val instanceof Map) {
|
||||
// merging at top level...
|
||||
Map<String,Object> newMap = (Map<String,Object>)val;
|
||||
handler.mergeMap(outer, newMap, path);
|
||||
} else {
|
||||
// todo: find a way to return query param in error message
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"Expected JSON Object but got " + val.getClass().getSimpleName() + "=" + val);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -270,6 +270,8 @@ public class RequestUtil {
|
|||
ObjectUtil.mergeObjects(json, path, o, handler);
|
||||
}
|
||||
}
|
||||
} catch (JSONParser.ParseException e ) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
|
||||
} catch (IOException e) {
|
||||
// impossible
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import javax.management.remote.JMXConnectorServerFactory;
|
|||
import javax.management.remote.JMXServiceURL;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
|
|
@ -31,12 +32,16 @@ import java.util.List;
|
|||
public final class JmxUtil {
|
||||
|
||||
/**
|
||||
* Retrieve the first MBeanServer found.
|
||||
* Retrieve the first MBeanServer found and if not found return the platform mbean server
|
||||
*
|
||||
* @return the first MBeanServer found
|
||||
*/
|
||||
public static MBeanServer findFirstMBeanServer() {
|
||||
return findMBeanServerForAgentId(null);
|
||||
MBeanServer mBeanServer = findMBeanServerForAgentId(null);
|
||||
if (mBeanServer == null) {
|
||||
return ManagementFactory.getPlatformMBeanServer();
|
||||
}
|
||||
return mBeanServer;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -19,12 +19,13 @@ package org.apache.solr.search.json;
|
|||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.solr.JSONTestUtil;
|
||||
import org.apache.solr.SolrTestCaseHS;
|
||||
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
|
||||
@LuceneTestCase.SuppressCodecs({"Lucene3x","Lucene40","Lucene41","Lucene42","Lucene45","Appending"})
|
||||
public class TestJsonRequest extends SolrTestCaseHS {
|
||||
|
||||
|
|
@ -79,6 +80,15 @@ public class TestJsonRequest extends SolrTestCaseHS {
|
|||
, "response/numFound==2"
|
||||
);
|
||||
|
||||
// invalid value
|
||||
SolrException ex = expectThrows(SolrException.class, () -> client.testJQ(params("q", "*:*", "json", "5")));
|
||||
assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code());
|
||||
|
||||
// this is to verify other json params are not affected
|
||||
client.testJQ( params("q", "cat_s:A", "json.limit", "1"),
|
||||
"response/numFound==2"
|
||||
);
|
||||
|
||||
// test multiple json params
|
||||
client.testJQ( params("json","{query:'cat_s:A'}", "json","{filter:'where_s:NY'}")
|
||||
, "response/numFound==1"
|
||||
|
|
|
|||
|
|
@ -93,6 +93,35 @@ If `true` then each stage of processing will be timed and a `timing` section wil
|
|||
`async`::
|
||||
Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>
|
||||
|
||||
`splitByPrefix`::
|
||||
If `true`, the split point will be selected by taking into account the distribution of compositeId values in the shard.
|
||||
A compositeId has the form `<prefix>!<suffix>`, where all documents with the same prefix are colocated on in the hash space.
|
||||
If there are multiple prefixes in the shard being split, then the split point will be selected to divide up the prefixes into as equal sized shards as possible without splitting any prefix.
|
||||
If there is only a single prefix in a shard, the range of the prefix will be divided in half.
|
||||
+
|
||||
The id field is usually scanned to determine the number of documents with each prefix.
|
||||
As an optimization, if an optional field called `id_prefix` exists and has the document prefix indexed (including the !) for each document,
|
||||
then that will be used to generate the counts.
|
||||
+
|
||||
One simple way to populate `id_prefix` is a copyField in the schema:
|
||||
[source,xml]
|
||||
----
|
||||
<!-- OPTIONAL, for optimization used by splitByPrefix if it exists -->
|
||||
<field name="id_prefix" type="composite_id_prefix" indexed="true" stored="false"/>
|
||||
<copyField source="id" dest="id_prefix"/>
|
||||
<fieldtype name="composite_id_prefix" class="solr.TextField">
|
||||
<analyzer>
|
||||
<tokenizer class="solr.PatternTokenizerFactory" pattern=".*!" group="0"/>
|
||||
</analyzer>
|
||||
</fieldtype>
|
||||
----
|
||||
|
||||
Current implementation details and limitations:
|
||||
|
||||
* Prefix size is calculated using number of documents with the prefix.
|
||||
* Only two level compositeIds are supported.
|
||||
* The shard can only be split into two.
|
||||
|
||||
=== SPLITSHARD Response
|
||||
|
||||
The output will include the status of the request and the new shard names, which will use the original shard as their basis, adding an underscore and a number. For example, "shard1" will become "shard1_0" and "shard1_1". If the status is anything other than "success", an error message will explain why the request failed.
|
||||
|
|
|
|||
|
|
@ -207,7 +207,7 @@ public class Lang {
|
|||
.withFunctionName("ttest", TTestEvaluator.class)
|
||||
.withFunctionName("pairedTtest", PairedTTestEvaluator.class)
|
||||
.withFunctionName("multiVariateNormalDistribution", MultiVariateNormalDistributionEvaluator.class)
|
||||
.withFunctionName("integrate", IntegrateEvaluator.class)
|
||||
.withFunctionName("integral", IntegrateEvaluator.class)
|
||||
.withFunctionName("density", DensityEvaluator.class)
|
||||
.withFunctionName("mannWhitney", MannWhitneyUEvaluator.class)
|
||||
.withFunctionName("sumSq", SumSqEvaluator.class)
|
||||
|
|
@ -300,6 +300,8 @@ public class Lang {
|
|||
.withFunctionName("upper", UpperEvaluator.class)
|
||||
.withFunctionName("split", SplitEvaluator.class)
|
||||
.withFunctionName("trim", TrimEvaluator.class)
|
||||
.withFunctionName("cosine", CosineDistanceEvaluator.class)
|
||||
.withFunctionName("trunc", TruncEvaluator.class)
|
||||
|
||||
// Boolean Stream Evaluators
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.client.solrj.io.eval;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.math3.exception.DimensionMismatchException;
|
||||
import org.apache.commons.math3.ml.distance.DistanceMeasure;
|
||||
import org.apache.commons.math3.util.Precision;
|
||||
import org.apache.solr.client.solrj.io.Tuple;
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
|
||||
|
||||
public class CosineDistanceEvaluator extends RecursiveEvaluator {
|
||||
protected static final long serialVersionUID = 1L;
|
||||
|
||||
public CosineDistanceEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
|
||||
super(expression, factory);
|
||||
}
|
||||
|
||||
public CosineDistanceEvaluator(StreamExpression expression, StreamFactory factory, List<String> ignoredNamedParameters) throws IOException{
|
||||
super(expression, factory, ignoredNamedParameters);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object evaluate(Tuple tuple) throws IOException {
|
||||
return new CosineDistance();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object doWork(Object... values) throws IOException {
|
||||
// Nothing to do here
|
||||
throw new IOException("This call should never occur");
|
||||
}
|
||||
|
||||
public static class CosineDistance implements DistanceMeasure {
|
||||
|
||||
private static final long serialVersionUID = -9108154600539125566L;
|
||||
|
||||
public double compute(double[] v1, double[] v2) throws DimensionMismatchException {
|
||||
return Precision.round(1-Math.abs(CosineSimilarityEvaluator.cosineSimilarity(v1, v2)), 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -20,6 +20,7 @@ import java.io.IOException;
|
|||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.apache.commons.math3.util.Precision;
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
|
||||
|
||||
|
|
@ -51,7 +52,7 @@ public class CosineSimilarityEvaluator extends RecursiveNumericEvaluator impleme
|
|||
return cosineSimilarity(d1, d2);
|
||||
}
|
||||
|
||||
private double cosineSimilarity(double[] vectorA, double[] vectorB) {
|
||||
public static double cosineSimilarity(double[] vectorA, double[] vectorB) {
|
||||
double dotProduct = 0.0;
|
||||
double normA = 0.0;
|
||||
double normB = 0.0;
|
||||
|
|
@ -60,7 +61,8 @@ public class CosineSimilarityEvaluator extends RecursiveNumericEvaluator impleme
|
|||
normA += Math.pow(vectorA[i], 2);
|
||||
normB += Math.pow(vectorB[i], 2);
|
||||
}
|
||||
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
||||
double d = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
||||
return Precision.round(d, 8);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import java.util.Locale;
|
|||
|
||||
import org.apache.commons.math3.analysis.DifferentiableUnivariateFunction;
|
||||
import org.apache.commons.math3.analysis.UnivariateFunction;
|
||||
import org.apache.commons.math3.analysis.interpolation.AkimaSplineInterpolator;
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
|
||||
|
||||
|
|
@ -42,12 +43,17 @@ public class DerivativeEvaluator extends RecursiveObjectEvaluator implements One
|
|||
}
|
||||
|
||||
VectorFunction vectorFunction = (VectorFunction) value;
|
||||
|
||||
DifferentiableUnivariateFunction func = null;
|
||||
double[] x = (double[])vectorFunction.getFromContext("x");
|
||||
|
||||
if(!(vectorFunction.getFunction() instanceof DifferentiableUnivariateFunction)) {
|
||||
throw new IOException("Cannot evaluate derivative from parameter.");
|
||||
double[] y = (double[])vectorFunction.getFromContext("y");
|
||||
func = new AkimaSplineInterpolator().interpolate(x, y);
|
||||
} else {
|
||||
func = (DifferentiableUnivariateFunction) vectorFunction.getFunction();
|
||||
}
|
||||
|
||||
DifferentiableUnivariateFunction func = (DifferentiableUnivariateFunction)vectorFunction.getFunction();
|
||||
double[] x = (double[])vectorFunction.getFromContext("x");
|
||||
UnivariateFunction derfunc = func.derivative();
|
||||
double[] dvalues = new double[x.length];
|
||||
for(int i=0; i<x.length; i++) {
|
||||
|
|
@ -56,7 +62,7 @@ public class DerivativeEvaluator extends RecursiveObjectEvaluator implements One
|
|||
|
||||
VectorFunction vf = new VectorFunction(derfunc, dvalues);
|
||||
vf.addToContext("x", x);
|
||||
vf.addToContext("y", vectorFunction.getFromContext("y"));
|
||||
vf.addToContext("y", dvalues);
|
||||
|
||||
return vf;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
package org.apache.solr.client.solrj.io.eval;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.apache.commons.math3.analysis.UnivariateFunction;
|
||||
|
|
@ -34,8 +35,8 @@ public class IntegrateEvaluator extends RecursiveObjectEvaluator implements Many
|
|||
@Override
|
||||
public Object doWork(Object... values) throws IOException {
|
||||
|
||||
if(values.length != 3) {
|
||||
throw new IOException("The integrate function requires 3 parameters");
|
||||
if(values.length > 3) {
|
||||
throw new IOException("The integrate function requires at most 3 parameters");
|
||||
}
|
||||
|
||||
if (!(values[0] instanceof VectorFunction)) {
|
||||
|
|
@ -43,28 +44,45 @@ public class IntegrateEvaluator extends RecursiveObjectEvaluator implements Many
|
|||
}
|
||||
|
||||
VectorFunction vectorFunction = (VectorFunction) values[0];
|
||||
if(!(vectorFunction.getFunction() instanceof UnivariateFunction)) {
|
||||
if (!(vectorFunction.getFunction() instanceof UnivariateFunction)) {
|
||||
throw new IOException("Cannot evaluate integral from parameter.");
|
||||
}
|
||||
|
||||
Number min = null;
|
||||
Number max = null;
|
||||
UnivariateFunction func = (UnivariateFunction) vectorFunction.getFunction();
|
||||
|
||||
if(values[1] instanceof Number) {
|
||||
min = (Number) values[1];
|
||||
if(values.length == 3) {
|
||||
|
||||
|
||||
Number min = null;
|
||||
Number max = null;
|
||||
|
||||
if (values[1] instanceof Number) {
|
||||
min = (Number) values[1];
|
||||
} else {
|
||||
throw new IOException("The second parameter of the integrate function must be a number");
|
||||
}
|
||||
|
||||
if (values[2] instanceof Number) {
|
||||
max = (Number) values[2];
|
||||
} else {
|
||||
throw new IOException("The third parameter of the integrate function must be a number");
|
||||
}
|
||||
|
||||
RombergIntegrator rombergIntegrator = new RombergIntegrator();
|
||||
return rombergIntegrator.integrate(5000, func, min.doubleValue(), max.doubleValue());
|
||||
} else {
|
||||
throw new IOException("The second parameter of the integrate function must be a number");
|
||||
RombergIntegrator integrator = new RombergIntegrator();
|
||||
|
||||
double[] x = (double[])vectorFunction.getFromContext("x");
|
||||
double[] y = (double[])vectorFunction.getFromContext("y");
|
||||
ArrayList<Number> out = new ArrayList();
|
||||
out.add(0);
|
||||
for(int i=1; i<x.length; i++) {
|
||||
out.add(integrator.integrate(5000, func, x[0], x[i]));
|
||||
}
|
||||
|
||||
return out;
|
||||
|
||||
}
|
||||
|
||||
if(values[2] instanceof Number ) {
|
||||
max = (Number) values[2];
|
||||
} else {
|
||||
throw new IOException("The third parameter of the integrate function must be a number");
|
||||
}
|
||||
|
||||
UnivariateFunction func = (UnivariateFunction)vectorFunction.getFunction();
|
||||
|
||||
RombergIntegrator rombergIntegrator = new RombergIntegrator();
|
||||
return rombergIntegrator.integrate(5000, func, min.doubleValue(), max.doubleValue());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -73,9 +73,11 @@ public class TopFeaturesEvaluator extends RecursiveObjectEvaluator implements Tw
|
|||
private List<Integer> getMaxIndexes(double[] values, int k) {
|
||||
TreeSet<Pair> set = new TreeSet();
|
||||
for(int i=0; i<values.length; i++) {
|
||||
set.add(new Pair(i, values[i]));
|
||||
if(set.size() > k) {
|
||||
set.pollFirst();
|
||||
if(values[i] > 0){
|
||||
set.add(new Pair(i, values[i]));
|
||||
if (set.size() > k) {
|
||||
set.pollFirst();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -89,16 +91,22 @@ public class TopFeaturesEvaluator extends RecursiveObjectEvaluator implements Tw
|
|||
|
||||
public static class Pair implements Comparable<Pair> {
|
||||
|
||||
private int index;
|
||||
private Integer index;
|
||||
private Double value;
|
||||
|
||||
public Pair(int index, Number value) {
|
||||
this.index = index;
|
||||
public Pair(int _index, Number value) {
|
||||
this.index = _index;
|
||||
this.value = value.doubleValue();
|
||||
}
|
||||
|
||||
public int compareTo(Pair pair) {
|
||||
return value.compareTo(pair.value);
|
||||
|
||||
int c = value.compareTo(pair.value);
|
||||
if(c==0) {
|
||||
return index.compareTo(pair.index);
|
||||
} else {
|
||||
return c;
|
||||
}
|
||||
}
|
||||
|
||||
public int getIndex() {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.client.solrj.io.eval;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
|
||||
|
||||
public class TruncEvaluator extends RecursiveObjectEvaluator implements TwoValueWorker {
|
||||
protected static final long serialVersionUID = 1L;
|
||||
|
||||
public TruncEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
|
||||
super(expression, factory);
|
||||
|
||||
if(2 != containedEvaluators.size()){
|
||||
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting exactly 2 values but found %d",expression,containedEvaluators.size()));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object doWork(Object value1, Object value2){
|
||||
if(null == value1){
|
||||
return null;
|
||||
}
|
||||
|
||||
int endIndex = ((Number)value2).intValue();
|
||||
|
||||
if(value1 instanceof List){
|
||||
return ((List<?>)value1).stream().map(innerValue -> doWork(innerValue, endIndex)).collect(Collectors.toList());
|
||||
}
|
||||
else {
|
||||
return value1.toString().substring(0, endIndex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
package org.apache.solr.client.solrj.response.json;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
|
|
@ -35,14 +36,14 @@ public class NestableJsonFacet {
|
|||
private long domainCount;
|
||||
private final Map<String, NestableJsonFacet> queryFacetsByName;
|
||||
private final Map<String, BucketBasedJsonFacet> bucketBasedFacetByName;
|
||||
private final Map<String, Number> statFacetsByName;
|
||||
private final Map<String, Object> statsByName;
|
||||
private final Map<String, HeatmapJsonFacet> heatmapFacetsByName;
|
||||
|
||||
public NestableJsonFacet(NamedList<Object> facetNL) {
|
||||
queryFacetsByName = new HashMap<>();
|
||||
bucketBasedFacetByName = new HashMap<>();
|
||||
statFacetsByName = new HashMap<>();
|
||||
heatmapFacetsByName = new HashMap<>();
|
||||
statsByName = new HashMap<>();
|
||||
|
||||
for (Map.Entry<String, Object> entry : facetNL) {
|
||||
final String key = entry.getKey();
|
||||
|
|
@ -50,8 +51,10 @@ public class NestableJsonFacet {
|
|||
continue;
|
||||
} else if ("count".equals(key)) {
|
||||
domainCount = ((Number) entry.getValue()).longValue();
|
||||
} else if(entry.getValue() instanceof Number) { // Stat/agg facet value
|
||||
statFacetsByName.put(key, (Number)entry.getValue());
|
||||
} else if (entry.getValue() instanceof Number || entry.getValue() instanceof String ||
|
||||
entry.getValue() instanceof Date) {
|
||||
// Stat/agg facet value
|
||||
statsByName.put(key, entry.getValue());
|
||||
} else if(entry.getValue() instanceof NamedList) { // Either heatmap/query/range/terms facet
|
||||
final NamedList<Object> facet = (NamedList<Object>) entry.getValue();
|
||||
final boolean isBucketBased = facet.get("buckets") != null;
|
||||
|
|
@ -103,17 +106,17 @@ public class NestableJsonFacet {
|
|||
}
|
||||
|
||||
/**
|
||||
* Retrieve the value for a stat or agg facet with the provided name
|
||||
* Retrieve the value for a stat or agg with the provided name
|
||||
*/
|
||||
public Number getStatFacetValue(String name) {
|
||||
return statFacetsByName.get(name);
|
||||
public Object getStatValue(String name) {
|
||||
return statsByName.get(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the names of any stat or agg facets that are direct descendants of this facet
|
||||
* @return the names of any stat or agg that are direct descendants of this facet
|
||||
*/
|
||||
public Set<String> getStatFacetNames() {
|
||||
return statFacetsByName.keySet();
|
||||
public Set<String> getStatNames() {
|
||||
return statsByName.keySet();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ package org.apache.solr.client.ref_guide_examples;
|
|||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -34,10 +35,10 @@ import org.apache.solr.client.solrj.request.json.JsonQueryRequest;
|
|||
import org.apache.solr.client.solrj.request.json.QueryFacetMap;
|
||||
import org.apache.solr.client.solrj.request.json.RangeFacetMap;
|
||||
import org.apache.solr.client.solrj.request.json.TermsFacetMap;
|
||||
import org.apache.solr.client.solrj.response.json.BucketJsonFacet;
|
||||
import org.apache.solr.client.solrj.response.json.NestableJsonFacet;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.client.solrj.response.UpdateResponse;
|
||||
import org.apache.solr.client.solrj.response.json.BucketJsonFacet;
|
||||
import org.apache.solr.client.solrj.response.json.NestableJsonFacet;
|
||||
import org.apache.solr.cloud.SolrCloudTestCase;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
|
|
@ -455,6 +456,7 @@ public class JsonRequestApiTest extends SolrCloudTestCase {
|
|||
.setQuery("memory")
|
||||
.withFilter("inStock:true")
|
||||
.withStatFacet("avg_price", "avg(price)")
|
||||
.withStatFacet("min_manufacturedate_dt", "min(manufacturedate_dt)")
|
||||
.withStatFacet("num_suppliers", "unique(manu_exact)")
|
||||
.withStatFacet("median_weight", "percentile(weight,50)");
|
||||
QueryResponse queryResponse = request.process(solrClient, COLLECTION_NAME);
|
||||
|
|
@ -464,9 +466,13 @@ public class JsonRequestApiTest extends SolrCloudTestCase {
|
|||
assertEquals(4, queryResponse.getResults().getNumFound());
|
||||
assertEquals(4, queryResponse.getResults().size());
|
||||
final NestableJsonFacet topLevelFacetingData = queryResponse.getJsonFacetingResponse();
|
||||
assertEquals(146.66, (double) topLevelFacetingData.getStatFacetValue("avg_price"), 0.5);
|
||||
assertEquals(3, topLevelFacetingData.getStatFacetValue("num_suppliers"));
|
||||
assertEquals(352.0, (double) topLevelFacetingData.getStatFacetValue("median_weight"), 0.5);
|
||||
assertEquals(146.66, (double) topLevelFacetingData.getStatValue("avg_price"), 0.5);
|
||||
assertEquals(3, topLevelFacetingData.getStatValue("num_suppliers"));
|
||||
assertEquals(352.0, (double) topLevelFacetingData.getStatValue("median_weight"), 0.5);
|
||||
|
||||
Object val = topLevelFacetingData.getStatValue("min_manufacturedate_dt");
|
||||
assertTrue(val instanceof Date);
|
||||
assertEquals("2006-02-13T15:26:37Z", ((Date)val).toInstant().toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -478,6 +484,7 @@ public class JsonRequestApiTest extends SolrCloudTestCase {
|
|||
.setQuery("*:*")
|
||||
.withFilter("price:[1.0 TO *]")
|
||||
.withFilter("popularity:[0 TO 10]")
|
||||
.withStatFacet("min_manu_id_s", "min(manu_id_s)")
|
||||
.withStatFacet("avg_value", "avg(div(popularity,price))");
|
||||
QueryResponse queryResponse = request.process(solrClient, COLLECTION_NAME);
|
||||
//end::solrj-json-metrics-facet-simple[]
|
||||
|
|
@ -486,7 +493,10 @@ public class JsonRequestApiTest extends SolrCloudTestCase {
|
|||
assertEquals(13, queryResponse.getResults().getNumFound());
|
||||
assertEquals(10, queryResponse.getResults().size());
|
||||
final NestableJsonFacet topLevelFacetingData = queryResponse.getJsonFacetingResponse();
|
||||
assertEquals(0.036, (double) topLevelFacetingData.getStatFacetValue("avg_value"), 0.1);
|
||||
assertEquals(0.036, (double) topLevelFacetingData.getStatValue("avg_value"), 0.1);
|
||||
Object val = topLevelFacetingData.getStatValue("min_manu_id_s");
|
||||
assertTrue(val instanceof String);
|
||||
assertEquals("apple", val.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -511,7 +521,7 @@ public class JsonRequestApiTest extends SolrCloudTestCase {
|
|||
assertEquals(13, queryResponse.getResults().getNumFound());
|
||||
assertEquals(10, queryResponse.getResults().size());
|
||||
final NestableJsonFacet topLevelFacetingData = queryResponse.getJsonFacetingResponse();
|
||||
assertEquals(0.108, (double) topLevelFacetingData.getStatFacetValue("avg_value"), 0.1);
|
||||
assertEquals(0.108, (double) topLevelFacetingData.getStatValue("avg_value"), 0.1);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -551,7 +561,7 @@ public class JsonRequestApiTest extends SolrCloudTestCase {
|
|||
assertEquals(10, queryResponse.getResults().size());
|
||||
final NestableJsonFacet topLevelFacetingData = queryResponse.getJsonFacetingResponse();
|
||||
assertEquals(2, topLevelFacetingData.getQueryFacet("high_popularity").getCount());
|
||||
assertEquals(199.5, topLevelFacetingData.getQueryFacet("high_popularity").getStatFacetValue("average_price"));
|
||||
assertEquals(199.5, topLevelFacetingData.getQueryFacet("high_popularity").getStatValue("average_price"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ public class TestLang extends SolrTestCase {
|
|||
"triangularDistribution", "precision", "minMaxScale", "markovChain", "grandSum",
|
||||
"scalarAdd", "scalarSubtract", "scalarMultiply", "scalarDivide", "sumRows",
|
||||
"sumColumns", "diff", "corrPValues", "normalizeSum", "geometricDistribution", "olsRegress",
|
||||
"derivative", "spline", "ttest", "pairedTtest", "multiVariateNormalDistribution", "integrate",
|
||||
"derivative", "spline", "ttest", "pairedTtest", "multiVariateNormalDistribution", "integral",
|
||||
"density", "mannWhitney", "sumSq", "akima", "lerp", "chiSquareDataSet", "gtestDataSet",
|
||||
"termVectors", "getColumnLabels", "getRowLabels", "getAttribute", "kmeans", "getCentroids",
|
||||
"getCluster", "topFeatures", "featureSelect", "rowAt", "colAt", "setColumnLabels",
|
||||
|
|
@ -77,7 +77,7 @@ public class TestLang extends SolrTestCase {
|
|||
"getSupportPoints", "pairSort", "log10", "plist", "recip", "pivot", "ltrim", "rtrim", "export",
|
||||
"zplot", "natural", "repeat", "movingMAD", "hashRollup", "noop", "var", "stddev", "recNum", "isNull",
|
||||
"notNull", "matches", "projectToBorder", "double", "long", "parseCSV", "parseTSV", "dateTime",
|
||||
"split", "upper", "trim", "lower"};
|
||||
"split", "upper", "trim", "lower", "trunc", "cosine"};
|
||||
|
||||
@Test
|
||||
public void testLang() {
|
||||
|
|
|
|||
|
|
@ -229,6 +229,27 @@ public class MathExpressionTest extends SolrCloudTestCase {
|
|||
assertEquals(s2, "c-d-hello");
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testTrunc() throws Exception {
|
||||
String expr = " select(list(tuple(field1=\"abcde\", field2=\"012345\")), trunc(field1, 2) as field3, trunc(field2, 4) as field4)";
|
||||
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
|
||||
paramsLoc.set("expr", expr);
|
||||
paramsLoc.set("qt", "/stream");
|
||||
|
||||
String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/" + COLLECTIONORALIAS;
|
||||
TupleStream solrStream = new SolrStream(url, paramsLoc);
|
||||
|
||||
StreamContext context = new StreamContext();
|
||||
solrStream.setStreamContext(context);
|
||||
List<Tuple> tuples = getTuples(solrStream);
|
||||
assertEquals(tuples.size(), 1);
|
||||
String s1 = tuples.get(0).getString("field3");
|
||||
assertEquals(s1, "ab");
|
||||
String s2 = tuples.get(0).getString("field4");
|
||||
assertEquals(s2, "0123");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUpperLowerSingle() throws Exception {
|
||||
String expr = " select(list(tuple(field1=\"a\", field2=\"C\")), upper(field1) as field3, lower(field2) as field4)";
|
||||
|
|
@ -249,6 +270,28 @@ public class MathExpressionTest extends SolrCloudTestCase {
|
|||
assertEquals(s2, "c");
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testTruncArray() throws Exception {
|
||||
String expr = " select(list(tuple(field1=array(\"aaaa\",\"bbbb\",\"cccc\"))), trunc(field1, 3) as field2)";
|
||||
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
|
||||
paramsLoc.set("expr", expr);
|
||||
paramsLoc.set("qt", "/stream");
|
||||
|
||||
String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/" + COLLECTIONORALIAS;
|
||||
TupleStream solrStream = new SolrStream(url, paramsLoc);
|
||||
|
||||
StreamContext context = new StreamContext();
|
||||
solrStream.setStreamContext(context);
|
||||
List<Tuple> tuples = getTuples(solrStream);
|
||||
assertEquals(tuples.size(), 1);
|
||||
List<String> l1 = (List<String>)tuples.get(0).get("field2");
|
||||
assertEquals(l1.get(0), "aaa");
|
||||
assertEquals(l1.get(1), "bbb");
|
||||
assertEquals(l1.get(2), "ccc");
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUpperLowerArray() throws Exception {
|
||||
String expr = " select(list(tuple(field1=array(\"a\",\"b\",\"c\"), field2=array(\"X\",\"Y\",\"Z\"))), upper(field1) as field3, lower(field2) as field4)";
|
||||
|
|
@ -722,6 +765,27 @@ public class MathExpressionTest extends SolrCloudTestCase {
|
|||
assertTrue(tuples.get(0).getDouble("cov").equals(-625.0D));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCosineDistance() throws Exception {
|
||||
String cexpr = "let(echo=true, " +
|
||||
"a=array(1,2,3,4)," +
|
||||
"b=array(10, 20, 30, 45), " +
|
||||
"c=distance(a, b, cosine()), " +
|
||||
")";
|
||||
|
||||
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
|
||||
paramsLoc.set("expr", cexpr);
|
||||
paramsLoc.set("qt", "/stream");
|
||||
String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/" + COLLECTIONORALIAS;
|
||||
TupleStream solrStream = new SolrStream(url, paramsLoc);
|
||||
StreamContext context = new StreamContext();
|
||||
solrStream.setStreamContext(context);
|
||||
List<Tuple> tuples = getTuples(solrStream);
|
||||
assertTrue(tuples.size() == 1);
|
||||
Number d = (Number) tuples.get(0).get("c");
|
||||
assertEquals(d.doubleValue(), 0.0017046159, 0.0001);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDistance() throws Exception {
|
||||
String cexpr = "let(echo=true, " +
|
||||
|
|
@ -3343,7 +3407,7 @@ public class MathExpressionTest extends SolrCloudTestCase {
|
|||
List<Tuple> tuples = getTuples(solrStream);
|
||||
assertTrue(tuples.size() == 1);
|
||||
Number cs = (Number)tuples.get(0).get("return-value");
|
||||
assertTrue(cs.doubleValue() == 0.9838197164968291);
|
||||
assertEquals(cs.doubleValue(),0.9838197164968291, .00000001);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -4085,9 +4149,10 @@ public class MathExpressionTest extends SolrCloudTestCase {
|
|||
String cexpr = "let(echo=true, " +
|
||||
"a=sequence(50, 1, 0), " +
|
||||
"b=spline(a), " +
|
||||
"c=integrate(b, 0, 49), " +
|
||||
"d=integrate(b, 0, 20), " +
|
||||
"e=integrate(b, 20, 49))";
|
||||
"c=integral(b, 0, 49), " +
|
||||
"d=integral(b, 0, 20), " +
|
||||
"e=integral(b, 20, 49)," +
|
||||
"f=integral(b))";
|
||||
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
|
||||
paramsLoc.set("expr", cexpr);
|
||||
paramsLoc.set("qt", "/stream");
|
||||
|
|
@ -4103,6 +4168,9 @@ public class MathExpressionTest extends SolrCloudTestCase {
|
|||
assertEquals(integral.doubleValue(), 20, 0.0);
|
||||
integral = (Number)tuples.get(0).get("e");
|
||||
assertEquals(integral.doubleValue(), 29, 0.0);
|
||||
List<Number> integrals = (List<Number>)tuples.get(0).get("f");
|
||||
assertEquals(integrals.size(), 50);
|
||||
assertEquals(integrals.get(49).intValue(), 49);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -4313,7 +4381,8 @@ public class MathExpressionTest extends SolrCloudTestCase {
|
|||
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@Test
|
||||
public void testLerp() throws Exception {
|
||||
String cexpr = "let(echo=true," +
|
||||
" a=array(0,1,2,3,4,5,6,7), " +
|
||||
|
|
|
|||
|
|
@ -545,8 +545,8 @@ public class DirectJsonQueryRequestFacetingIntegrationTest extends SolrCloudTest
|
|||
|
||||
private void assertHasStatFacetWithValue(NestableJsonFacet response, String expectedFacetName, Double expectedStatValue) {
|
||||
assertTrue("Expected response to have stat facet named '" + expectedFacetName + "'",
|
||||
response.getStatFacetValue(expectedFacetName) != null);
|
||||
assertEquals(expectedStatValue, response.getStatFacetValue(expectedFacetName));
|
||||
response.getStatValue(expectedFacetName) != null);
|
||||
assertEquals(expectedStatValue, response.getStatValue(expectedFacetName));
|
||||
}
|
||||
|
||||
private void assertExpectedDocumentsFoundAndReturned(QueryResponse response, int expectedNumFound, int expectedReturned) {
|
||||
|
|
|
|||
|
|
@ -571,8 +571,8 @@ public class JsonQueryRequestFacetingIntegrationTest extends SolrCloudTestCase {
|
|||
|
||||
private void assertHasStatFacetWithValue(NestableJsonFacet response, String expectedFacetName, Double expectedStatValue) {
|
||||
assertTrue("Expected response to have stat facet named '" + expectedFacetName + "'",
|
||||
response.getStatFacetValue(expectedFacetName) != null);
|
||||
assertEquals(expectedStatValue, response.getStatFacetValue(expectedFacetName));
|
||||
response.getStatValue(expectedFacetName) != null);
|
||||
assertEquals(expectedStatValue, response.getStatValue(expectedFacetName));
|
||||
}
|
||||
|
||||
private void assertExpectedDocumentsFoundAndReturned(QueryResponse response, int expectedNumFound, int expectedReturned) {
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ import org.apache.solr.common.util.ExecutorUtil;
|
|||
import org.apache.solr.common.util.ObjectReleaseTracker;
|
||||
import org.apache.solr.common.util.SolrjNamedThreadFactory;
|
||||
import org.apache.solr.common.util.SuppressForbidden;
|
||||
import org.apache.solr.common.util.TimeSource;
|
||||
import org.apache.solr.common.util.Utils;
|
||||
import org.apache.solr.common.util.XML;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
|
|
@ -152,6 +153,7 @@ import org.apache.solr.util.SSLTestConfig;
|
|||
import org.apache.solr.util.StartupLoggingUtils;
|
||||
import org.apache.solr.util.TestHarness;
|
||||
import org.apache.solr.util.TestInjection;
|
||||
import org.apache.solr.util.TimeOut;
|
||||
import org.apache.zookeeper.KeeperException;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
|
|
@ -3116,7 +3118,7 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
|
|||
QueryResponse rsp = client.query(collection, solrQuery);
|
||||
long found = rsp.getResults().getNumFound();
|
||||
|
||||
if (rsp.getResults().getNumFound() == expectedDocCount) {
|
||||
if (found == expectedDocCount) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -3131,9 +3133,17 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
|
|||
// Add the bogus doc
|
||||
new UpdateRequest().add(bogus).commit(client, collection);
|
||||
|
||||
// Let's spin until we find the doc.
|
||||
checkUniqueDoc(client, collection, idField, bogusID, true);
|
||||
|
||||
// Then remove it, we should be OK now since new searchers have been opened.
|
||||
new UpdateRequest().deleteById(bogusID).commit(client, collection);
|
||||
// Let's check again to see if we succeeded
|
||||
|
||||
// Now spin until the doc is gone.
|
||||
checkUniqueDoc(client, collection, idField, bogusID, false);
|
||||
|
||||
// At this point we're absolutely, totally, positive that a new searcher has been opened, so go ahead and check
|
||||
// the actual condition.
|
||||
rsp = client.query(collection, solrQuery);
|
||||
found = rsp.getResults().getNumFound();
|
||||
|
||||
|
|
@ -3145,6 +3155,31 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
|
|||
} else if (failAnyway) {
|
||||
fail("Solr11035BandAid failAnyway == true, would have successfully repaired the collection: '" + collection
|
||||
+ "' extra info: '" + tag + "'");
|
||||
} else {
|
||||
log.warn("Solr11035BandAid, repair successful");
|
||||
}
|
||||
}
|
||||
// Helper for bandaid
|
||||
private static void checkUniqueDoc(SolrClient client, String collection, String idField, String id, boolean shouldBeThere) throws IOException, SolrServerException {
|
||||
TimeOut timeOut = new TimeOut(100, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
||||
final SolrQuery solrQuery = new SolrQuery(idField + ":" + id);
|
||||
|
||||
while (!timeOut.hasTimedOut()) {
|
||||
QueryResponse rsp = client.query(collection, solrQuery);
|
||||
long found = rsp.getResults().getNumFound();
|
||||
if (shouldBeThere && found == 1) {
|
||||
return;
|
||||
}
|
||||
if (shouldBeThere == false && found == 0) {
|
||||
return;
|
||||
}
|
||||
log.warn("Solr11035BandAid should have succeeded in checkUniqueDoc, shouldBeThere == {}, numFound = {}. Will try again after 250 ms sleep", shouldBeThere, found);
|
||||
try {
|
||||
Thread.sleep(250);
|
||||
} catch (InterruptedException e) {
|
||||
return; // just bail
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue