SOLR-2452: Merged with trunk up to r1137125

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/solr2452@1137126 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Steven Rowe 2011-06-18 05:16:57 +00:00
commit 4c9c9259e8
35 changed files with 1768 additions and 829 deletions

View File

@ -22,6 +22,7 @@
<orderEntry type="library" name="Solr library" level="project" />
<orderEntry type="library" name="Solr example library" level="project" />
<orderEntry type="module" module-name="spatial" />
<orderEntry type="module" module-name="grouping" />
<orderEntry type="module" module-name="highlighter" />
<orderEntry type="module" module-name="icu" />
<orderEntry type="module" module-name="queries" />

View File

@ -34,7 +34,6 @@
<properties>
<module-directory>solr/contrib/analysis-extras</module-directory>
<build-directory>../../build/contrib/analysis-extras</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
<dependency>

View File

@ -34,7 +34,6 @@
<properties>
<module-directory>solr/contrib/clustering</module-directory>
<build-directory>../../build/contrib/clustering</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
<dependency>

View File

@ -34,7 +34,6 @@
<properties>
<module-directory>solr/contrib/dataimporthandler-extras</module-directory>
<build-directory>../../build/contrib/dataimporthandler-extras</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
<dependency>

View File

@ -34,7 +34,6 @@
<properties>
<module-directory>solr/contrib/dataimporthandler</module-directory>
<build-directory>../../build/contrib/dataimporthandler</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
<dependency>

View File

@ -37,7 +37,6 @@
<properties>
<module-directory>solr/contrib/extraction</module-directory>
<build-directory>../../build/contrib/extraction</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
<dependency>

View File

@ -34,7 +34,6 @@
<properties>
<module-directory>solr/contrib/uima</module-directory>
<build-directory>../../build/contrib/uima</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
<dependency>

View File

@ -39,7 +39,7 @@
<module>contrib</module>
</modules>
<properties>
<java.compat.version>1.6</java.compat.version>
<tests.luceneMatchVersion>LUCENE_CURRENT</tests.luceneMatchVersion>
</properties>
<issueManagement>
<system>JIRA</system>

View File

@ -34,7 +34,6 @@
<properties>
<module-directory>solr</module-directory>
<build-directory>../build</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
<dependency>
@ -92,6 +91,11 @@
<artifactId>lucene-suggest</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-grouping</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-commons-csv</artifactId>

View File

@ -34,7 +34,6 @@
<properties>
<module-directory>solr/src/solrj</module-directory>
<build-directory>../../build/solrj</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
<dependency>

View File

@ -543,6 +543,12 @@ New Features
* LUCENE-3191: Added TopDocs.merge, to facilitate merging results from
different shards (Uwe Schindler, Mike McCandless)
* LUCENE-3179: Added OpenBitSet.prevSetBit (Paul Elschot via Mike McCandless)
* LUCENE-3210: Made TieredMergePolicy more aggressive in reclaiming
segments with deletions; added new methods
set/getReclaimDeletesWeight to control this. (Mike McCandless)
Build
* LUCENE-1344: Create OSGi bundle using dev-tools/maven.

View File

@ -79,6 +79,10 @@ New Features
facilitate doing grouping in a distributed environment (Uwe
Schindler, Mike McCandless)
* LUCENE-2919: Added PKIndexSplitter, that splits an index according
to a middle term in a specified field. (Jason Rutherglen via Mike
McCandless)
API Changes
* LUCENE-3141: add getter method to access fragInfos in FieldFragList.

View File

@ -0,0 +1,136 @@
package org.apache.lucene.index;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.Version;
/**
* Split an index based on a given primary key term
* and a 'middle' term. If the middle term is present, it's
* sent to dir2.
*/
public class PKIndexSplitter {
private Term midTerm;
Directory input;
Directory dir1;
Directory dir2;
public PKIndexSplitter(Term midTerm, Directory input,
Directory dir1, Directory dir2) {
this.midTerm = midTerm;
this.input = input;
this.dir1 = dir1;
this.dir2 = dir2;
}
public void split() throws IOException {
IndexReader reader = IndexReader.open(input);
OpenBitSet lowDels = setDeletes(reader, null, midTerm.bytes());
OpenBitSet hiDels = setDeletes(reader, midTerm.bytes(), null);
createIndex(dir1, reader, lowDels);
createIndex(dir2, reader, hiDels);
reader.close();
}
private void createIndex(Directory target, IndexReader reader, OpenBitSet bv) throws IOException {
IndexWriter w = new IndexWriter(target, new IndexWriterConfig(
Version.LUCENE_CURRENT,
new WhitespaceAnalyzer(Version.LUCENE_CURRENT))
.setOpenMode(OpenMode.CREATE));
w.addIndexes(new DeletesIndexReader(reader, bv));
w.close();
}
private OpenBitSet setDeletes(IndexReader reader, BytesRef startTerm,
BytesRef endTermExcl) throws IOException {
OpenBitSet incl = new OpenBitSet(reader.maxDoc());
Terms terms = MultiFields.getTerms(reader, midTerm.field());
TermsEnum te = terms.iterator();
if (startTerm != null) {
te.seek(startTerm);
}
while (true) {
final BytesRef term = te.next();
if (term == null) {
break;
}
if (endTermExcl != null && term.compareTo(endTermExcl) >= 0) {
break;
}
DocsEnum docs = MultiFields.getTermDocsEnum(reader,
MultiFields.getDeletedDocs(reader), midTerm.field(), term);
while (true) {
final int doc = docs.nextDoc();
if (doc != DocsEnum.NO_MORE_DOCS) {
incl.set(doc);
} else break;
}
}
OpenBitSet dels = new OpenBitSet(reader.maxDoc());
for (int x=0; x < reader.maxDoc(); x++) {
if (!incl.get(x)) {
dels.set(x);
}
}
return dels;
}
public static class DeletesIndexReader extends FilterIndexReader {
OpenBitSet readerDels;
public DeletesIndexReader(IndexReader reader, OpenBitSet deletes) {
super(new SlowMultiReaderWrapper(reader));
readerDels = new OpenBitSet(reader.maxDoc());
if (in.hasDeletions()) {
final Bits oldDelBits = MultiFields.getDeletedDocs(in);
assert oldDelBits != null;
for (int i = 0; i < in.maxDoc(); i++) {
if (oldDelBits.get(i) || deletes.get(i)) {
readerDels.set(i);
}
}
} else {
readerDels = deletes;
}
}
@Override
public int numDocs() {
return in.maxDoc() - (int)readerDels.cardinality();
}
@Override
public boolean hasDeletions() {
return (int)readerDels.cardinality() > 0;
}
@Override
public Bits getDeletedDocs() {
return readerDels;
}
}
}

View File

@ -0,0 +1,93 @@
package org.apache.lucene.index;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
import java.text.DecimalFormat;
import java.text.NumberFormat;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.LuceneTestCase;
public class TestPKIndexSplitter extends LuceneTestCase {
public void testSplit() throws Exception {
NumberFormat format = new DecimalFormat("000000000");
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(
Version.LUCENE_CURRENT,
new WhitespaceAnalyzer(Version.LUCENE_CURRENT))
.setOpenMode(OpenMode.CREATE));
for (int x=0; x < 10; x++) {
Document doc = createDocument(x, "1", 3, format);
w.addDocument(doc);
}
for (int x=15; x < 20; x++) {
Document doc = createDocument(x, "2", 3, format);
w.addDocument(doc);
}
w.close();
Directory dir1 = newDirectory();
Directory dir2 = newDirectory();
Term splitTerm = new Term("id", new BytesRef(format.format(11)));
PKIndexSplitter splitter = new PKIndexSplitter(splitTerm,
dir, dir1, dir2);
splitter.split();
IndexReader ir1 = IndexReader.open(dir1);
IndexReader ir2 = IndexReader.open(dir2);
assertEquals(10, ir1.maxDoc());
assertEquals(4, ir2.maxDoc());
ir1.close();
ir2.close();
dir1.close();
dir2.close();
dir.close();
}
public Document createDocument(int n, String indexName,
int numFields, NumberFormat format) {
StringBuilder sb = new StringBuilder();
Document doc = new Document();
String id = format.format(n);
doc.add(new Field("id", id, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
doc.add(new Field("indexname", indexName, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
sb.append("a");
sb.append(n);
doc.add(new Field("field1", sb.toString(), Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
sb.append(" b");
sb.append(n);
for (int i = 1; i < numFields; i++) {
doc.add(new Field("field" + (i + 1), sb.toString(), Store.YES,
Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
}
return doc;
}
}

View File

@ -30,7 +30,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.values.IndexDocValues;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;

View File

@ -82,6 +82,7 @@ public class TieredMergePolicy extends MergePolicy {
private double expungeDeletesPctAllowed = 10.0;
private boolean useCompoundFile = true;
private double noCFSRatio = 0.1;
private double reclaimDeletesWeight = 2.0;
/** Maximum number of segments to be merged at a time
* during "normal" merging. For explicit merging (eg,
@ -133,6 +134,23 @@ public class TieredMergePolicy extends MergePolicy {
return maxMergedSegmentBytes/1024/1024.;
}
/** Controls how aggressively merges that reclaim more
* deletions are favored. Higher values favor selecting
* merges that reclaim deletions. A value of 0.0 means
* deletions don't impact merge selection. */
public TieredMergePolicy setReclaimDeletesWeight(double v) {
if (v < 0.0) {
throw new IllegalArgumentException("reclaimDeletesWeight must be >= 0.0 (got " + v + ")");
}
reclaimDeletesWeight = v;
return this;
}
/** See {@link #setReclaimDeletesWeight}. */
public double getReclaimDeletesWeight() {
return reclaimDeletesWeight;
}
/** Segments smaller than this are "rounded up" to this
* size, ie treated as equal (floor) size for merge
* selection. This is to prevent frequent flushing of
@ -435,7 +453,7 @@ public class TieredMergePolicy extends MergePolicy {
// Strongly favor merges that reclaim deletes:
final double nonDelRatio = ((double) totAfterMergeBytes)/totBeforeMergeBytes;
mergeScore *= nonDelRatio;
mergeScore *= Math.pow(nonDelRatio, reclaimDeletesWeight);
final double finalMergeScore = mergeScore;

View File

@ -17,22 +17,22 @@ package org.apache.lucene.search;
* limitations under the License.
*/
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.util.RamUsageEstimator;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.util.RamUsageEstimator;
/**
* Caches all docs, and optionally also scores, coming from
* a search, and is then able to replay them to another
* collector. You specify the max RAM this class may use.
* Once the collection is done, call {@link #isCached}. If
* this returns true, you can use {@link #replay} against a
* new collector. If it returns false, this means too much
* RAM was required and you must instead re-run the original
* search.
* Once the collection is done, call {@link #isCached}. If
* this returns true, you can use {@link #replay(Collector)}
* against a new collector. If it returns false, this means
* too much RAM was required and you must instead re-run the
* original search.
*
* <p><b>NOTE</b>: this class consumes 4 (or 8 bytes, if
* scoring is cached) per collected document. If the result
@ -105,7 +105,16 @@ public abstract class CachingCollector extends Collector {
cachedScorer = new CachedScorer();
cachedScores = new ArrayList<float[]>();
curScores = new float[128];
curScores = new float[INITIAL_ARRAY_SIZE];
cachedScores.add(curScores);
}
ScoreCachingCollector(Collector other, int maxDocsToCache) {
super(other, maxDocsToCache);
cachedScorer = new CachedScorer();
cachedScores = new ArrayList<float[]>();
curScores = new float[INITIAL_ARRAY_SIZE];
cachedScores.add(curScores);
}
@ -210,7 +219,11 @@ public abstract class CachingCollector extends Collector {
NoScoreCachingCollector(Collector other, double maxRAMMB) {
super(other, maxRAMMB, false);
}
NoScoreCachingCollector(Collector other, int maxDocsToCache) {
super(other, maxDocsToCache);
}
@Override
public void collect(int doc) throws IOException {
@ -353,7 +366,25 @@ public abstract class CachingCollector extends Collector {
*/
public static CachingCollector create(Collector other, boolean cacheScores, double maxRAMMB) {
return cacheScores ? new ScoreCachingCollector(other, maxRAMMB) : new NoScoreCachingCollector(other, maxRAMMB);
}
}
/**
* Create a new {@link CachingCollector} that wraps the given collector and
* caches documents and scores up to the specified max docs threshold.
*
* @param other
* the Collector to wrap and delegate calls to.
* @param cacheScores
* whether to cache scores in addition to document IDs. Note that
* this increases the RAM consumed per doc
* @param maxDocsToCache
* the maximum number of documents for caching the documents and
* possible the scores. If the collector exceeds the threshold,
* no documents and scores are cached.
*/
public static CachingCollector create(Collector other, boolean cacheScores, int maxDocsToCache) {
return cacheScores ? new ScoreCachingCollector(other, maxDocsToCache) : new NoScoreCachingCollector(other, maxDocsToCache);
}
// Prevent extension from non-internal classes
private CachingCollector(Collector other, double maxRAMMB, boolean cacheScores) {
@ -369,6 +400,15 @@ public abstract class CachingCollector extends Collector {
}
maxDocsToCache = (int) ((maxRAMMB * 1024 * 1024) / bytesPerDoc);
}
private CachingCollector(Collector other, int maxDocsToCache) {
this.other = other;
cachedDocs = new ArrayList<int[]>();
curDocs = new int[INITIAL_ARRAY_SIZE];
cachedDocs.add(curDocs);
this.maxDocsToCache = maxDocsToCache;
}
@Override
public boolean acceptsDocsOutOfOrder() {

View File

@ -778,6 +778,28 @@ public final class BitUtil {
return n - (y & 1);
}
/** table of number of leading zeros in a byte */
public static final byte[] nlzTable = {8,7,6,6,5,5,5,5,4,4,4,4,4,4,4,4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
/** Returns the number of leading zero bits.
*/
public static int nlz(long x) {
int n = 0;
// do the first step as a long
int y = (int)(x>>>32);
if (y==0) {n+=32; y = (int)(x); }
if ((y & 0xFFFF0000) == 0) { n+=16; y<<=16; }
if ((y & 0xFF000000) == 0) { n+=8; y<<=8; }
return n + nlzTable[y >>> 24];
/* implementation without table:
if ((y & 0xF0000000) == 0) { n+=4; y<<=4; }
if ((y & 0xC0000000) == 0) { n+=2; y<<=2; }
if ((y & 0x80000000) == 0) { n+=1; y<<=1; }
if ((y & 0x80000000) == 0) { n+=1;}
return n;
*/
}
/** returns true if v is a power of two or zero*/
public static boolean isPowerOfTwo(int v) {

View File

@ -659,7 +659,34 @@ public class OpenBitSet extends DocIdSet implements Bits, Cloneable {
}
/** Returns the index of the first set bit starting downwards at
* the index specified.
* -1 is returned if there are no more set bits.
*/
public int prevSetBit(int index) {
if (index < 0) {
return -1;
}
int i = index>>6;
if (i >= wlen) {
i = wlen - 1;
}
final int subIndex = index & 0x3f; // index within the word
long word = (bits[i] << (63-subIndex)); // skip all the bits to the left of index
if (word != 0) {
return (i << 6) + subIndex - Long.numberOfLeadingZeros(word); // See LUCENE-3197
}
while (--i >= 0) {
word = bits[i];
if (word !=0 ) {
return (i << 6) + 63 - Long.numberOfLeadingZeros(word);
}
}
return -1;
}
@Override
public Object clone() {

View File

@ -947,6 +947,7 @@ public abstract class LuceneTestCase extends Assert {
tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 2, 20));
tmp.setUseCompoundFile(r.nextBoolean());
tmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8);
tmp.setReclaimDeletesWeight(r.nextDouble()*4);
return tmp;
}

View File

@ -17,15 +17,11 @@ package org.apache.lucene.search;
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.CachingCollector;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.LuceneTestCase;
import java.io.IOException;
public class TestCachingCollector extends LuceneTestCase {
private static final double ONE_BYTE = 1.0 / (1024 * 1024); // 1 byte out of MB
@ -76,7 +72,7 @@ public class TestCachingCollector extends LuceneTestCase {
public void testBasic() throws Exception {
for (boolean cacheScores : new boolean[] { false, true }) {
CachingCollector cc = CachingCollector.create(new NoOpCollector(false), cacheScores, 1);
CachingCollector cc = CachingCollector.create(new NoOpCollector(false), cacheScores, 1.0);
cc.setScorer(new MockScorer());
// collect 1000 docs

View File

@ -0,0 +1,133 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util;
public class TestBitUtil extends LuceneTestCase {
private static int slowNlz(long x) {
if (x == 0L) return 64;
int nlz = 0;
while ( ((x << nlz) & (1L << 63)) == 0) {
nlz++;
}
return nlz;
}
private void checkNlz(long x) {
assertEquals(slowNlz(x), BitUtil.nlz(x));
assertEquals(Long.numberOfLeadingZeros(x), BitUtil.nlz(x));
}
public void testNlz() {
checkNlz(0L);
checkNlz(1L);
checkNlz(-1L);
for (int i = 1; i <= 63; i++) {
checkNlz(1L << i);
checkNlz((1L << i) + (1L << (i>>1)));
}
}
public void testBitUtils() {
long num = 100000;
assertEquals( 5, BitUtil.ntz(num) );
assertEquals( 5, BitUtil.ntz2(num) );
assertEquals( 5, BitUtil.ntz3(num) );
num = 10;
assertEquals( 1, BitUtil.ntz(num) );
assertEquals( 1, BitUtil.ntz2(num) );
assertEquals( 1, BitUtil.ntz3(num) );
for (int i=0; i<64; i++) {
num = 1L << i;
assertEquals( i, BitUtil.ntz(num) );
assertEquals( i, BitUtil.ntz2(num) );
assertEquals( i, BitUtil.ntz3(num) );
}
}
private long testArg(int shift) {
return (1L << shift) + (1L << (shift>>1));
}
private long nlzBitUtilBasicLoop(int iters) {
long sumRes = 0;
while (iters-- >= 0) {
for (int i = 1; i <= 63; i++) {
long a = testArg(i);
sumRes += BitUtil.nlz(a);
sumRes += BitUtil.nlz(a+1);
sumRes += BitUtil.nlz(a-1);
sumRes += BitUtil.nlz(a+10);
sumRes += BitUtil.nlz(a-10);
}
}
return sumRes;
}
private long nlzLongBasicLoop(int iters) {
long sumRes = 0;
while (iters-- >= 0) {
for (int i = 1; i <= 63; i++) {
long a = testArg(i);
sumRes += Long.numberOfLeadingZeros(a);
sumRes += Long.numberOfLeadingZeros(a+1);
sumRes += Long.numberOfLeadingZeros(a-1);
sumRes += Long.numberOfLeadingZeros(a+10);
sumRes += Long.numberOfLeadingZeros(a-10);
}
}
return sumRes;
}
public void tstPerfNlz() { // See LUCENE-3197, prefer to use Long.numberOfLeadingZeros() over BitUtil.nlz().
final long measureMilliSecs = 2000;
final int basicIters = 100000;
long startTime;
long endTime;
long curTime;
long dummy = 0; // avoid optimizing away
dummy = 0;
int bitUtilLoops = 0;
startTime = System.currentTimeMillis();
endTime = startTime + measureMilliSecs;
do {
dummy += nlzBitUtilBasicLoop(basicIters);
bitUtilLoops++;
curTime = System.currentTimeMillis();
} while (curTime < endTime);
int bitUtilPsTime = (int) (1000000000 * (curTime - startTime) / (basicIters * 5 * 63 * (float) bitUtilLoops));
System.out.println("BitUtil nlz time: " + (bitUtilPsTime/1) + " picosec/call, dummy: " + dummy);
dummy = 0;
int longLoops = 0;
startTime = System.currentTimeMillis();
endTime = startTime + measureMilliSecs;
do {
dummy += nlzLongBasicLoop(basicIters);
longLoops++;
curTime = System.currentTimeMillis();
} while (curTime < endTime);
int longPsTime = (int) (1000000000 * (curTime - startTime) / (basicIters * 5 * 63 * (float) longLoops));
System.out.println("Long nlz time: " + longPsTime + " picosec/call, dummy: " + dummy);
}
}

View File

@ -41,6 +41,20 @@ public class TestOpenBitSet extends LuceneTestCase {
} while (aa>=0);
}
void doPrevSetBit(BitSet a, OpenBitSet b) {
int aa=a.length();
int bb=aa;
do {
// aa = a.prevSetBit(aa-1);
aa--;
while ((aa >= 0) && (! a.get(aa))) {
aa--;
}
bb = b.prevSetBit(bb-1);
assertEquals(aa,bb);
} while (aa>=0);
}
// test interleaving different OpenBitSetIterator.next()/skipTo()
void doIterate(BitSet a, OpenBitSet b, int mode) {
if (mode==1) doIterate1(a, b);
@ -123,6 +137,7 @@ public class TestOpenBitSet extends LuceneTestCase {
bb = (OpenBitSet)b.clone(); bb.clear(fromIndex,toIndex);
doNextSetBit(aa,bb); // a problem here is from clear() or nextSetBit
doPrevSetBit(aa,bb);
fromIndex = random.nextInt(sz+80);
toIndex = fromIndex + random.nextInt((sz>>1)+1);
@ -130,6 +145,7 @@ public class TestOpenBitSet extends LuceneTestCase {
bb = (OpenBitSet)b.clone(); bb.set(fromIndex,toIndex);
doNextSetBit(aa,bb); // a problem here is from set() or nextSetBit
doPrevSetBit(aa,bb);
if (a0 != null) {
@ -168,7 +184,7 @@ public class TestOpenBitSet extends LuceneTestCase {
b0=b;
}
}
// large enough to flush obvious bugs, small enough to run in <.5 sec as part of a
// larger testsuite.
public void testSmall() {
@ -176,12 +192,13 @@ public class TestOpenBitSet extends LuceneTestCase {
doRandomSets(atLeast(1200), atLeast(1000), 2);
}
// uncomment to run a bigger test (~2 minutes).
/*
public void testBig() {
// uncomment to run a bigger test (~2 minutes).
// rand = newRandom();
// doRandomSets(2000,200000, 1);
// doRandomSets(2000,200000, 2);
doRandomSets(2000,200000, 1);
doRandomSets(2000,200000, 2);
}
*/
public void testEquals() {
OpenBitSet b1 = new OpenBitSet(1111);
@ -205,26 +222,6 @@ public class TestOpenBitSet extends LuceneTestCase {
assertFalse(b1.equals(new Object()));
}
public void testBitUtils()
{
long num = 100000;
assertEquals( 5, BitUtil.ntz(num) );
assertEquals( 5, BitUtil.ntz2(num) );
assertEquals( 5, BitUtil.ntz3(num) );
num = 10;
assertEquals( 1, BitUtil.ntz(num) );
assertEquals( 1, BitUtil.ntz2(num) );
assertEquals( 1, BitUtil.ntz3(num) );
for (int i=0; i<64; i++) {
num = 1L << i;
assertEquals( i, BitUtil.ntz(num) );
assertEquals( i, BitUtil.ntz2(num) );
assertEquals( i, BitUtil.ntz3(num) );
}
}
public void testHashCodeEquals() {
OpenBitSet bs1 = new OpenBitSet(200);
OpenBitSet bs2 = new OpenBitSet(64);
@ -233,6 +230,35 @@ public class TestOpenBitSet extends LuceneTestCase {
assertEquals(bs1, bs2);
assertEquals(bs1.hashCode(), bs2.hashCode());
}
private OpenBitSet makeOpenBitSet(int[] a) {
OpenBitSet bs = new OpenBitSet();
for (int e: a) {
bs.set(e);
}
return bs;
}
private BitSet makeBitSet(int[] a) {
BitSet bs = new BitSet();
for (int e: a) {
bs.set(e);
}
return bs;
}
private void checkPrevSetBitArray(int [] a) {
OpenBitSet obs = makeOpenBitSet(a);
BitSet bs = makeBitSet(a);
doPrevSetBit(bs, obs);
}
public void testPrevSetBit() {
checkPrevSetBitArray(new int[] {});
checkPrevSetBitArray(new int[] {0});
checkPrevSetBitArray(new int[] {0,2});
}
}

View File

@ -78,6 +78,8 @@ New Features
Bojan Smid, Charles Hornberger, Dieter Grad, Dmitry Lihachev, Doug Steigerwald,
Karsten Sperling, Michael Gundlach, Oleg Gnatovskiy, Thomas Traeger,
Harish Agarwal, yonik)
SOLR-2564: Integrate grouping module into Solr. Also adds the ability to return number of
groups that have match a query.
* SOLR-1665: Add debug component options for timings, results and query info only (gsingers, hossman, yonik)

View File

@ -49,7 +49,7 @@
<target name="compile-core" depends="init, clover, compile-analyzers-common,
compile-analyzers-phonetic, compile-suggest, compile-highlighter,
compile-memory, compile-misc, compile-queries, compile-spatial,
compile-solrj, common.compile-core, compile-webapp"
compile-grouping, compile-solrj, common.compile-core, compile-webapp"
unless="solr-core.compiled">
</target>
@ -150,6 +150,9 @@
<target name="compile-suggest" unless="suggest.uptodate">
<ant dir="${common.dir}/../modules/suggest" target="default" inheritAll="false"/>
</target>
<target name="compile-grouping" unless="grouping.uptodate">
<ant dir="${common.dir}/../modules/grouping" target="default" inheritAll="false"/>
</target>
<target name="compile-highlighter" unless="highlighter.uptodate">
<ant dir="${common.dir}/contrib/highlighter" target="default" inheritAll="false"/>
</target>

View File

@ -60,6 +60,8 @@
property="analyzers-phonetic.uptodate" classpath.property="analyzers-phonetic.jar"/>
<module-uptodate name="suggest" jarfile="${common.dir}/../modules/suggest/build/lucene-suggest-${version}.jar"
property="suggest.uptodate" classpath.property="suggest.jar"/>
<module-uptodate name="grouping" jarfile="${common.dir}/../modules/grouping/build/lucene-grouping-${version}.jar"
property="grouping.uptodate" classpath.property="grouping.jar"/>
<contrib-uptodate name="highlighter" property="highlighter.uptodate" classpath.property="highlighter.jar"/>
<contrib-uptodate name="memory" property="memory.uptodate" classpath.property="memory.jar"/>
<contrib-uptodate name="misc" property="misc.uptodate" classpath.property="misc.jar"/>
@ -75,6 +77,7 @@
<pathelement path="${queries.jar}"/>
<pathelement path="${spatial.jar}"/>
<pathelement path="${suggest.jar}"/>
<pathelement path="${grouping.jar}"/>
<pathelement location="${common-solr.dir}/build/classes/solrj"/>
<pathelement location="${common-solr.dir}/build/classes/webapp"/>
<pathelement location="${common-solr.dir}/build/classes/java"/>

View File

@ -120,7 +120,7 @@ public class TestSqlEntityProcessorDelta extends AbstractDataImportHandlerTestCa
runFullImport(dataConfig_delta);
assertQ(req("id:1"), "//*[@numFound='0']");
} finally {
f.setWritable(true);
f.delete();
}
}
}

View File

@ -38,5 +38,20 @@ public interface GroupParams {
/** treat the first group result as the main result. true/false */
public static final String GROUP_FORMAT = GROUP + ".format";
/**
* Whether to cache the first pass search (doc ids and score) for the second pass search.
* Also defines the maximum size of the group cache relative to maxdoc in a percentage.
* Values can be a positive integer, from 0 till 100. A value of 0 will disable the group cache.
* The default is 0.*/
public static final String GROUP_CACHE_PERCENTAGE = GROUP + ".cache.percent";
// Note: Since you can supply multiple fields to group on, but only have a facets for the whole result. It only makes
// sense to me to support these parameters for the first group.
/** Whether the docSet (for example for faceting) should be based on plain documents (a.k.a UNGROUPED) or on the groups (a.k.a GROUPED). */
public static final String GROUP_COLLAPSE = GROUP + ".collapse";
/** Whether the group count should be included in the response. */
public static final String GROUP_TOTAL_COUNT = GROUP + ".ngroups";
}

View File

@ -41,7 +41,9 @@ public class QuerySenderListener extends AbstractSolrEventListener {
public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher) {
final SolrIndexSearcher searcher = newSearcher;
log.info("QuerySenderListener sending requests to " + newSearcher);
for (NamedList nlst : (List<NamedList>)args.get("queries")) {
List<NamedList> allLists = (List<NamedList>)args.get("queries");
if (allLists == null) return;
for (NamedList nlst : allLists) {
SolrQueryRequest req = null;
try {

View File

@ -45,8 +45,6 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.*;
import org.apache.solr.search.function.FunctionQuery;
import org.apache.solr.search.function.QueryValueSource;
import org.apache.solr.util.SolrPluginUtils;
import java.io.IOException;
@ -315,16 +313,25 @@ public class QueryComponent extends SearchComponent
boolean doGroup = params.getBool(GroupParams.GROUP, false);
if (doGroup) {
try {
Grouping grouping = new Grouping(searcher, result, cmd);
int maxDocsPercentageToCache = params.getInt(GroupParams.GROUP_CACHE_PERCENTAGE, 0);
boolean cacheSecondPassSearch = maxDocsPercentageToCache >= 1 && maxDocsPercentageToCache <= 100;
String[] fields = params.getParams(GroupParams.GROUP_FIELD);
String[] funcs = params.getParams(GroupParams.GROUP_FUNC);
String[] queries = params.getParams(GroupParams.GROUP_QUERY);
String groupSortStr = params.get(GroupParams.GROUP_SORT);
boolean main = params.getBool(GroupParams.GROUP_MAIN, false);
String format = params.get(GroupParams.GROUP_FORMAT);
Grouping.Format defaultFormat = "simple".equals(format) ? Grouping.Format.Simple : Grouping.Format.Grouped;
String formatStr = params.get(GroupParams.GROUP_FORMAT, Grouping.Format.grouped.name());
Grouping.Format defaultFormat;
try {
defaultFormat = Grouping.Format.valueOf(formatStr);
} catch (IllegalArgumentException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.format("Illegal %s parameter", GroupParams.GROUP_FORMAT));
}
boolean includeTotalGroupCount = params.getBool(GroupParams.GROUP_TOTAL_COUNT, false);
Grouping.TotalCount defaultTotalCount = includeTotalGroupCount ? Grouping.TotalCount.grouped : Grouping.TotalCount.ungrouped;
Sort sort = cmd.getSort();
// groupSort defaults to sort
Sort groupSort = groupSortStr == null ? cmd.getSort() : QueryParsing.parseSort(groupSortStr, req);
@ -332,95 +339,47 @@ public class QueryComponent extends SearchComponent
int groupOffsetDefault = params.getInt(GroupParams.GROUP_OFFSET, 0);
int docsPerGroupDefault = params.getInt(GroupParams.GROUP_LIMIT, 1);
// temporary: implement all group-by-field as group-by-func
if (funcs == null) {
funcs = fields;
} else if (fields != null) {
// catenate functions and fields
String[] both = new String[fields.length + funcs.length];
System.arraycopy(fields, 0, both, 0, fields.length);
System.arraycopy(funcs, 0, both, fields.length, funcs.length);
funcs = both;
}
Grouping grouping = new Grouping(searcher, result, cmd, cacheSecondPassSearch, maxDocsPercentageToCache, main);
grouping.setSort(sort)
.setGroupSort(groupSort)
.setDefaultFormat(defaultFormat)
.setLimitDefault(limitDefault)
.setDefaultTotalCount(defaultTotalCount)
.setDocsPerGroupDefault(docsPerGroupDefault)
.setGroupOffsetDefault(groupOffsetDefault);
if (fields != null) {
for (String field : fields) {
grouping.addFieldCommand(field, rb.req);
}
}
if (funcs != null) {
for (String groupByStr : funcs) {
QParser parser = QParser.getParser(groupByStr, "func", rb.req);
Query q = parser.getQuery();
Grouping.CommandFunc gc = grouping.new CommandFunc();
gc.groupSort = groupSort;
if (q instanceof FunctionQuery) {
gc.groupBy = ((FunctionQuery)q).getValueSource();
} else {
gc.groupBy = new QueryValueSource(q, 0.0f);
}
gc.key = groupByStr;
gc.numGroups = limitDefault;
gc.docsPerGroup = docsPerGroupDefault;
gc.groupOffset = groupOffsetDefault;
gc.offset = cmd.getOffset();
gc.sort = cmd.getSort();
gc.format = defaultFormat;
if (main) {
gc.main = true;
gc.format = Grouping.Format.Simple;
main = false;
}
if (gc.format == Grouping.Format.Simple) {
gc.groupOffset = 0; // doesn't make sense
}
grouping.add(gc);
grouping.addFunctionCommand(groupByStr, rb.req);
}
}
if (queries != null) {
for (String groupByStr : queries) {
QParser parser = QParser.getParser(groupByStr, null, rb.req);
Query gq = parser.getQuery();
Grouping.CommandQuery gc = grouping.new CommandQuery();
gc.query = gq;
gc.groupSort = groupSort;
gc.key = groupByStr;
gc.numGroups = limitDefault;
gc.docsPerGroup = docsPerGroupDefault;
gc.groupOffset = groupOffsetDefault;
// these two params will only be used if this is for the main result set
gc.offset = cmd.getOffset();
gc.numGroups = limitDefault;
gc.format = defaultFormat;
if (main) {
gc.main = true;
gc.format = Grouping.Format.Simple;
main = false;
}
if (gc.format == Grouping.Format.Simple) {
gc.docsPerGroup = gc.numGroups; // doesn't make sense to limit to one
gc.groupOffset = gc.offset;
}
grouping.add(gc);
grouping.addQueryCommand(groupByStr, rb.req);
}
}
if (rb.doHighlights || rb.isDebug()) {
// we need a single list of the returned docs
cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
}
// searcher.search(result,cmd);
grouping.execute();
rb.setResult( result );
if (grouping.isSignalCacheWarning()) {
rsp.add(
"cacheWarning",
String.format("Cache limit of %d percent relative to maxdoc has exceeded. Please increase cache size or disable caching.", maxDocsPercentageToCache)
);
}
rb.setResult(result);
rsp.add("grouped", result.groupedResults);
// TODO: get "hits" a different way to log
if (grouping.mainResult != null) {
ResultContext ctx = new ResultContext();
@ -428,10 +387,10 @@ public class QueryComponent extends SearchComponent
ctx.query = null; // TODO? add the query?
rsp.add("response", ctx);
rsp.getToLog().add("hits", grouping.mainResult.matches());
} else if (!grouping.getCommands().isEmpty()) { // Can never be empty since grouping.execute() checks for this.
rsp.getToLog().add("hits", grouping.getCommands().get(0).getMatches());
}
return;
} catch (ParseException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}

File diff suppressed because it is too large Load Diff

View File

@ -37,6 +37,10 @@ public abstract class FieldCacheSource extends ValueSource {
return cache;
}
public String getField() {
return field;
}
@Override
public String description() {
return field;

View File

@ -0,0 +1,79 @@
<?xml version="1.0" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- $Id: solrconfig-querysender.xml 1048886 2010-12-14 01:10:52Z hossman $
$Source$
$Name$
-->
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
<!-- The DirectoryFactory to use for indexes.
solr.StandardDirectoryFactory, the default, is filesystem based.
solr.RAMDirectoryFactory is memory based and not persistent. -->
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
<updateHandler class="solr.DirectUpdateHandler2">
<listener event="postCommit"
class="org.apache.solr.core.MockEventListener" />
<listener event="postOptimize"
class="org.apache.solr.core.MockEventListener" />
</updateHandler>
<query>
<!-- a newSearcher event is fired whenever a new searcher is being prepared
and there is a current searcher handling requests (aka registered). -->
<!-- QuerySenderListener takes an array of NamedList and executes a
local query request for each NamedList in sequence. -->
<listener event="newSearcher" class="solr.QuerySenderListener">
<!--
<arr name="queries">
<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> <str name="qt">mock</str></lst>
<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> <str name="qt">mock</str></lst>
</arr>
-->
</listener>
<listener event="newSearcher"
class="org.apache.solr.core.MockEventListener" />
<!-- a firstSearcher event is fired whenever a new searcher is being
prepared but there is no current registered searcher to handle
requests or to gain prewarming data from. -->
<listener event="firstSearcher" class="solr.QuerySenderListener">
<!--
<arr name="queries">
<lst> <str name="q">fast_warm</str> <str name="start">0</str> <str name="rows">10</str>
<str name="qt">mock</str>
</lst>
</arr>
-->
</listener>
<listener event="firstSearcher"
class="org.apache.solr.core.MockEventListener" />
</query>
<requestHandler name="mock" class="org.apache.solr.core.MockQuerySenderListenerReqHandler" default="true">
<!-- default values for query parameters -->
</requestHandler>
</config>

View File

@ -20,6 +20,7 @@ package org.apache.solr;
import org.apache.lucene.search.FieldCache;
import org.apache.noggit.JSONUtil;
import org.apache.noggit.ObjectBuilder;
import org.apache.solr.common.params.GroupParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.IndexSchema;
import org.junit.Before;
@ -88,6 +89,49 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
);
}
@Test
public void testGroupingGroupSortingScore_withTotalGroupCount() {
assertU(add(doc("id", "1","name", "author1", "title", "a book title", "group_si", "1")));
assertU(add(doc("id", "2","name", "author1", "title", "the title", "group_si", "2")));
assertU(add(doc("id", "3","name", "author2", "title", "a book title", "group_si", "1")));
assertU(add(doc("id", "4","name", "author2", "title", "title", "group_si", "2")));
assertU(add(doc("id", "5","name", "author3", "title", "the title of a title", "group_si", "1")));
assertU(commit());
assertQ(req("q","title:title", "group", "true", "group.field","name", "group.ngroups", "true")
,"//lst[@name='grouped']/lst[@name='name']"
,"//lst[@name='grouped']/lst[@name='name']/int[@name='matches'][.='5']"
,"//lst[@name='grouped']/lst[@name='name']/int[@name='ngroups'][.='3']"
,"*[count(//arr[@name='groups']/lst) = 3]"
,"//arr[@name='groups']/lst[1]/str[@name='groupValue'][.='author2']"
,"//arr[@name='groups']/lst[1]/result[@numFound='2']"
,"//arr[@name='groups']/lst[1]/result/doc/*[@name='id'][.='4']"
,"//arr[@name='groups']/lst[2]/str[@name='groupValue'][.='author1']"
,"//arr[@name='groups']/lst[2]/result[@numFound='2']"
,"//arr[@name='groups']/lst[2]/result/doc/*[@name='id'][.='2']"
,"//arr[@name='groups']/lst[3]/str[@name='groupValue'][.='author3']"
,"//arr[@name='groups']/lst[3]/result[@numFound='1']"
,"//arr[@name='groups']/lst[3]/result/doc/*[@name='id'][.='5']"
);
assertQ(req("q","title:title", "group", "true", "group.field","group_si", "group.ngroups", "true")
,"//lst[@name='grouped']/lst[@name='group_si']/int[@name='matches'][.='5']"
,"//lst[@name='grouped']/lst[@name='group_si']/int[@name='ngroups'][.='2']"
,"*[count(//arr[@name='groups']/lst) = 2]"
,"//arr[@name='groups']/lst[1]/int[@name='groupValue'][.='2']"
,"//arr[@name='groups']/lst[1]/result[@numFound='2']"
,"//arr[@name='groups']/lst[1]/result/doc/*[@name='id'][.='4']"
,"//arr[@name='groups']/lst[2]/int[@name='groupValue'][.='1']"
,"//arr[@name='groups']/lst[2]/result[@numFound='3']"
,"//arr[@name='groups']/lst[2]/result/doc/*[@name='id'][.='5']"
);
}
@Test
public void testGroupingGroupSortingScore_basicWithGroupSortEqualToSort() {
assertU(add(doc("id", "1","name", "author1", "title", "a book title")));
@ -353,7 +397,7 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
, "/grouped/foo_i=={'matches':10,'doclist':"
+"{'numFound':10,'start':1,'docs':[{'id':'10'},{'id':'3'},{'id':'6'}]}}"
);
};
}
@ -476,14 +520,16 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
List<Grp> sortedGroups = new ArrayList(groups.values());
Collections.sort(sortedGroups, groupComparator==sortComparator ? createFirstDocComparator(sortComparator) : createMaxDocComparator(sortComparator));
Object modelResponse = buildGroupedResult(h.getCore().getSchema(), sortedGroups, start, rows, group_offset, group_limit);
boolean includeNGroups = random.nextBoolean();
Object modelResponse = buildGroupedResult(h.getCore().getSchema(), sortedGroups, start, rows, group_offset, group_limit, includeNGroups);
int randomPercentage = random.nextInt(101);
// TODO: create a random filter too
SolrQueryRequest req = req("group","true","wt","json","indent","true", "echoParams","all", "q","{!func}score_f", "group.field",groupField
,sortStr==null ? "nosort":"sort", sortStr ==null ? "": sortStr
,(groupSortStr==null || groupSortStr==sortStr) ? "nosort":"group.sort", groupSortStr==null ? "": groupSortStr
,"rows",""+rows, "start",""+start, "group.offset",""+group_offset, "group.limit",""+group_limit
,(groupSortStr==null || groupSortStr==sortStr) ? "noGroupsort":"group.sort", groupSortStr==null ? "": groupSortStr
,"rows",""+rows, "start",""+start, "group.offset",""+group_offset, "group.limit",""+group_limit,
GroupParams.GROUP_CACHE_PERCENTAGE, Integer.toString(randomPercentage), GroupParams.GROUP_TOTAL_COUNT, includeNGroups ? "true" : "false"
);
String strResponse = h.query(req);
@ -508,7 +554,7 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
}
public static Object buildGroupedResult(IndexSchema schema, List<Grp> sortedGroups, int start, int rows, int group_offset, int group_limit) {
public static Object buildGroupedResult(IndexSchema schema, List<Grp> sortedGroups, int start, int rows, int group_offset, int group_limit, boolean includeNGroups) {
Map<String,Object> result = new LinkedHashMap<String,Object>();
long matches = 0;
@ -516,6 +562,9 @@ public class TestGroupingSearch extends SolrTestCaseJ4 {
matches += grp.docs.size();
}
result.put("matches", matches);
if (includeNGroups) {
result.put("ngroups", sortedGroups.size());
}
List groupList = new ArrayList();
result.put("groups", groupList);

View File

@ -0,0 +1,90 @@
package org.apache.solr.core;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.store.Directory;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.EventParams;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.TestExtendedDismaxParser;
import org.apache.solr.util.RefCounted;
import org.junit.BeforeClass;
import org.junit.Test;
public class TestQuerySenderNoQuery extends SolrTestCaseJ4 {
// number of instances configured in the solrconfig.xml
private static final int EXPECTED_MOCK_LISTENER_INSTANCES = 4;
private static int preInitMockListenerCount = 0;
@BeforeClass
public static void beforeClass() throws Exception {
// record current value prior to core initialization
// so we can verify the correct number of instances later
// NOTE: this won't work properly if concurrent tests run
// in the same VM
preInitMockListenerCount = MockEventListener.getCreateCount();
initCore("solrconfig-querysender-noquery.xml","schema.xml");
}
public void testListenerCreationCounts() {
SolrCore core = h.getCore();
assertEquals("Unexpected number of listeners created",
EXPECTED_MOCK_LISTENER_INSTANCES,
MockEventListener.getCreateCount() - preInitMockListenerCount);
}
@Test
public void testRequestHandlerRegistry() {
// property values defined in build.xml
SolrCore core = h.getCore();
assertEquals( 2, core.firstSearcherListeners.size() );
assertEquals( 2, core.newSearcherListeners.size() );
}
// Determine that when the query lists are commented out of both new and
// first searchers in the config, we don't throw an NPE
@Test
public void testSearcherEvents() throws Exception {
SolrCore core = h.getCore();
SolrEventListener newSearcherListener = core.newSearcherListeners.get(0);
assertTrue("Not an instance of QuerySenderListener", newSearcherListener instanceof QuerySenderListener);
QuerySenderListener qsl = (QuerySenderListener) newSearcherListener;
RefCounted<SolrIndexSearcher> currentSearcherRef = core.getSearcher();
SolrIndexSearcher currentSearcher = currentSearcherRef.get();
SolrIndexSearcher dummy = null;
qsl.newSearcher(currentSearcher, dummy);//test first Searcher (since param is null)
MockQuerySenderListenerReqHandler mock = (MockQuerySenderListenerReqHandler) core.getRequestHandler("mock");
assertNotNull("Mock is null", mock);
assertNull("Req (firstsearcher) is not null", mock.req);
Directory dir = currentSearcher.getIndexReader().directory();
SolrIndexSearcher newSearcher = new SolrIndexSearcher(core, core.getSchema(), "testQuerySenderNoQuery", dir, true, false);
qsl.newSearcher(newSearcher, currentSearcher); // get newSearcher.
assertNull("Req (newsearcher) is not null", mock.req);
newSearcher.close();
currentSearcherRef.decref();
}
}