diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index a9a2504532d..052e890f04f 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -200,10 +200,19 @@ Bug fixes * LUCENE-5668: Fix off-by-one in TieredMergePolicy (Mike McCandless) +* LUCENE-5673: MMapDirectory: Work around a "bug" in the JDK that throws + a confusing OutOfMemoryError wrapped inside IOException if the FileChannel + mapping failed because of lack of virtual address space. The IOException is + rethrown with more useful information about the problem, omitting the + incorrect OutOfMemoryError. (Robert Muir, Uwe Schindler) + Test Framework * LUCENE-5622: Fail tests if they print over the given limit of bytes to System.out or System.err. (Robert Muir, Dawid Weiss) + +* LUCENE-5619: Added backwards compatibility tests to ensure we can update existing + indexes with doc-values updates. (Shai Erera, Robert Muir) ======================= Lucene 4.8.0 ======================= diff --git a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java index abcf09728d8..8cb0eeb3376 100644 --- a/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java +++ b/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java @@ -28,10 +28,10 @@ import org.apache.lucene.analysis.icu.tokenattributes.ScriptAttribute; import com.ibm.icu.lang.UScript; import java.io.IOException; -import java.io.Reader; import java.io.StringReader; import java.util.Arrays; import java.util.Random; +import java.util.concurrent.CountDownLatch; public class TestICUTokenizer extends BaseTokenStreamTestCase { @@ -270,4 +270,43 @@ public class TestICUTokenizer extends BaseTokenStreamTestCase { ts.end(); } } + + /** test for bugs like http://bugs.icu-project.org/trac/ticket/10767 */ + public void testICUConcurrency() throws Exception { + int numThreads = 8; + final CountDownLatch startingGun = new CountDownLatch(1); + Thread threads[] = new Thread[numThreads]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread() { + @Override + public void run() { + try { + startingGun.await(); + long tokenCount = 0; + final String contents = "英 เบียร์ ビール ເບຍ abc"; + for (int i = 0; i < 1000; i++) { + try (Tokenizer tokenizer = new ICUTokenizer()) { + tokenizer.setReader(new StringReader(contents)); + tokenizer.reset(); + while (tokenizer.incrementToken()) { + tokenCount++; + } + tokenizer.end(); + } + } + if (VERBOSE) { + System.out.println(tokenCount); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }; + threads[i].start(); + } + startingGun.countDown(); + for (int i = 0; i < threads.length; i++) { + threads[i].join(); + } + } } diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StreamUtils.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StreamUtils.java index 6a3dd3cfb81..fbcec4d784d 100644 --- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StreamUtils.java +++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StreamUtils.java @@ -56,17 +56,14 @@ public class StreamUtils { try { return csfType==null ? in : new CompressorStreamFactory().createCompressorInputStream(csfType, in); } catch (CompressorException e) { - IOException ioe = new IOException(e.getMessage()); - ioe.initCause(e); - throw ioe; } + throw new IOException(e.getMessage(), e); + } } private OutputStream outputStream(OutputStream os) throws IOException { try { return csfType==null ? os : new CompressorStreamFactory().createCompressorOutputStream(csfType, os); } catch (CompressorException e) { - IOException ioe = new IOException(e.getMessage()); - ioe.initCause(e); - throw ioe; + throw new IOException(e.getMessage(), e); } } } diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java index 07a804a46e8..55fb1ee052a 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java @@ -168,9 +168,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer { try { bd = (BigDecimal) decoder.parse(scratch.utf8ToString()); } catch (ParseException pe) { - CorruptIndexException e = new CorruptIndexException("failed to parse BigDecimal value (resource=" + in + ")"); - e.initCause(pe); - throw e; + throw new CorruptIndexException("failed to parse BigDecimal value (resource=" + in + ")", pe); } SimpleTextUtil.readLine(in, scratch); // read the line telling us if its real or not return BigInteger.valueOf(field.minValue).add(bd.toBigIntegerExact()).longValue(); @@ -231,9 +229,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer { try { len = decoder.parse(new String(scratch.bytes, scratch.offset + LENGTH.length, scratch.length - LENGTH.length, StandardCharsets.UTF_8)).intValue(); } catch (ParseException pe) { - CorruptIndexException e = new CorruptIndexException("failed to parse int length (resource=" + in + ")"); - e.initCause(pe); - throw e; + throw new CorruptIndexException("failed to parse int length (resource=" + in + ")", pe); } result.bytes = new byte[len]; result.offset = 0; @@ -263,9 +259,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer { try { len = decoder.parse(new String(scratch.bytes, scratch.offset + LENGTH.length, scratch.length - LENGTH.length, StandardCharsets.UTF_8)).intValue(); } catch (ParseException pe) { - CorruptIndexException e = new CorruptIndexException("failed to parse int length (resource=" + in + ")"); - e.initCause(pe); - throw e; + throw new CorruptIndexException("failed to parse int length (resource=" + in + ")", pe); } // skip past bytes byte bytes[] = new byte[len]; @@ -310,9 +304,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer { try { return (int) ordDecoder.parse(scratch.utf8ToString()).longValue()-1; } catch (ParseException pe) { - CorruptIndexException e = new CorruptIndexException("failed to parse ord (resource=" + in + ")"); - e.initCause(pe); - throw e; + throw new CorruptIndexException("failed to parse ord (resource=" + in + ")", pe); } } catch (IOException ioe) { throw new RuntimeException(ioe); @@ -332,9 +324,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer { try { len = decoder.parse(new String(scratch.bytes, scratch.offset + LENGTH.length, scratch.length - LENGTH.length, StandardCharsets.UTF_8)).intValue(); } catch (ParseException pe) { - CorruptIndexException e = new CorruptIndexException("failed to parse int length (resource=" + in + ")"); - e.initCause(pe); - throw e; + throw new CorruptIndexException("failed to parse int length (resource=" + in + ")", pe); } result.bytes = new byte[len]; result.offset = 0; @@ -410,9 +400,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer { try { len = decoder.parse(new String(scratch.bytes, scratch.offset + LENGTH.length, scratch.length - LENGTH.length, StandardCharsets.UTF_8)).intValue(); } catch (ParseException pe) { - CorruptIndexException e = new CorruptIndexException("failed to parse int length (resource=" + in + ")"); - e.initCause(pe); - throw e; + throw new CorruptIndexException("failed to parse int length (resource=" + in + ")", pe); } result.bytes = new byte[len]; result.offset = 0; diff --git a/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java b/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java index 583a4bac7ef..ce495ea08ce 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java +++ b/lucene/core/src/java/org/apache/lucene/index/CorruptIndexException.java @@ -24,8 +24,13 @@ import java.io.IOException; * an inconsistency in the index. */ public class CorruptIndexException extends IOException { - /** Sole constructor. */ + /** Create exception with a message only */ public CorruptIndexException(String message) { super(message); } + + /** Create exception with message and root cause. */ + public CorruptIndexException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index a6275e1164b..a0d4c71f925 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -1708,11 +1708,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit{ for(int i=0;i + * @see Blog post about MMapDirectory */ public class MMapDirectory extends FSDirectory { private boolean useUnmapHack = UNMAP_SUPPORTED; @@ -216,7 +218,7 @@ public class MMapDirectory extends FSDirectory { private final boolean useUnmapHack; MMapIndexInput(String resourceDescription, FileChannel fc) throws IOException { - super(resourceDescription, map(fc, 0, fc.size()), fc.size(), chunkSizePower, getUseUnmap()); + super(resourceDescription, map(resourceDescription, fc, 0, fc.size()), fc.size(), chunkSizePower, getUseUnmap()); this.useUnmapHack = getUseUnmap(); } @@ -244,18 +246,16 @@ public class MMapDirectory extends FSDirectory { } }); } catch (PrivilegedActionException e) { - final IOException ioe = new IOException("unable to unmap the mapped buffer"); - ioe.initCause(e.getCause()); - throw ioe; + throw new IOException("Unable to unmap the mapped buffer: " + toString(), e.getCause()); } } } } /** Maps a file into a set of buffers */ - ByteBuffer[] map(FileChannel fc, long offset, long length) throws IOException { + final ByteBuffer[] map(String resourceDescription, FileChannel fc, long offset, long length) throws IOException { if ((length >>> chunkSizePower) >= Integer.MAX_VALUE) - throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + fc.toString()); + throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + resourceDescription); final long chunkSize = 1L << chunkSizePower; @@ -270,10 +270,45 @@ public class MMapDirectory extends FSDirectory { ? chunkSize : (length - bufferStart) ); - buffers[bufNr] = fc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize); + try { + buffers[bufNr] = fc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize); + } catch (IOException ioe) { + throw convertMapFailedIOException(ioe, resourceDescription, bufSize); + } bufferStart += bufSize; } return buffers; } + + private IOException convertMapFailedIOException(IOException ioe, String resourceDescription, int bufSize) { + final String originalMessage; + final Throwable originalCause; + if (ioe.getCause() instanceof OutOfMemoryError) { + // nested OOM confuses users, because its "incorrect", just print a plain message: + originalMessage = "Map failed"; + originalCause = null; + } else { + originalMessage = ioe.getMessage(); + originalCause = ioe.getCause(); + } + final String moreInfo; + if (!Constants.JRE_IS_64BIT) { + moreInfo = "MMapDirectory should only be used on 64bit platforms, because the address space on 32bit operating systems is too small. "; + } else if (Constants.WINDOWS) { + moreInfo = "Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. "; + } else if (Constants.LINUX) { + moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'), and 'sysctl vm.max_map_count'. "; + } else { + moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'). "; + } + final IOException newIoe = new IOException(String.format(Locale.ENGLISH, + "%s: %s [this may be caused by lack of enough unfragmented virtual address space "+ + "or too restrictive virtual memory limits enforced by the operating system, "+ + "preventing us to map a chunk of %d bytes. %sMore information: "+ + "http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]", + originalMessage, resourceDescription, bufSize, moreInfo), originalCause); + newIoe.setStackTrace(ioe.getStackTrace()); + return newIoe; + } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index 76128ba9bc4..723a8d992d0 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -162,6 +162,64 @@ public class TestBackwardsCompatibility extends LuceneTestCase { } */ + private void updateNumeric(IndexWriter writer, String id, String f, String cf, long value) throws IOException { + writer.updateNumericDocValue(new Term("id", id), f, value); + writer.updateNumericDocValue(new Term("id", id), cf, value*2); + } + + private void updateBinary(IndexWriter writer, String id, String f, String cf, long value) throws IOException { + writer.updateBinaryDocValue(new Term("id", id), f, TestBinaryDocValuesUpdates.toBytes(value)); + writer.updateBinaryDocValue(new Term("id", id), cf, TestBinaryDocValuesUpdates.toBytes(value*2)); + } + +/* // Creates an index with DocValues updates + public void testCreateIndexWithDocValuesUpdates() throws Exception { + // we use a real directory name that is not cleaned up, + // because this method is only used to create backwards + // indexes: + File indexDir = new File("/tmp/idx/dvupdates"); + TestUtil.rm(indexDir); + Directory dir = newFSDirectory(indexDir); + + IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) + .setUseCompoundFile(false).setMergePolicy(NoMergePolicy.INSTANCE); + IndexWriter writer = new IndexWriter(dir, conf); + // create an index w/ few doc-values fields, some with updates and some without + for (int i = 0; i < 30; i++) { + Document doc = new Document(); + doc.add(new StringField("id", "" + i, Store.NO)); + doc.add(new NumericDocValuesField("ndv1", i)); + doc.add(new NumericDocValuesField("ndv1_c", i*2)); + doc.add(new NumericDocValuesField("ndv2", i*3)); + doc.add(new NumericDocValuesField("ndv2_c", i*6)); + doc.add(new BinaryDocValuesField("bdv1", TestBinaryDocValuesUpdates.toBytes(i))); + doc.add(new BinaryDocValuesField("bdv1_c", TestBinaryDocValuesUpdates.toBytes(i*2))); + doc.add(new BinaryDocValuesField("bdv2", TestBinaryDocValuesUpdates.toBytes(i*3))); + doc.add(new BinaryDocValuesField("bdv2_c", TestBinaryDocValuesUpdates.toBytes(i*6))); + writer.addDocument(doc); + if ((i+1) % 10 == 0) { + writer.commit(); // flush every 10 docs + } + } + + // first segment: no updates + + // second segment: update two fields, same gen + updateNumeric(writer, "10", "ndv1", "ndv1_c", 100L); + updateBinary(writer, "11", "bdv1", "bdv1_c", 100L); + writer.commit(); + + // third segment: update few fields, different gens, few docs + updateNumeric(writer, "20", "ndv1", "ndv1_c", 100L); + updateBinary(writer, "21", "bdv1", "bdv1_c", 100L); + writer.commit(); + updateNumeric(writer, "22", "ndv1", "ndv1_c", 200L); // update the field again + writer.commit(); + + writer.shutdown(); + dir.close(); + }*/ + final static String[] oldNames = {"40.cfs", "40.nocfs", "41.cfs", @@ -983,4 +1041,62 @@ public class TestBackwardsCompatibility extends LuceneTestCase { TestUtil.checkIndex(dir); dir.close(); } + + public static final String dvUpdatesIndex = "dvupdates.48.zip"; + + private void assertNumericDocValues(AtomicReader r, String f, String cf) throws IOException { + NumericDocValues ndvf = r.getNumericDocValues(f); + NumericDocValues ndvcf = r.getNumericDocValues(cf); + for (int i = 0; i < r.maxDoc(); i++) { + assertEquals(ndvcf.get(i), ndvf.get(i)*2); + } + } + + private void assertBinaryDocValues(AtomicReader r, String f, String cf) throws IOException { + BinaryDocValues bdvf = r.getBinaryDocValues(f); + BinaryDocValues bdvcf = r.getBinaryDocValues(cf); + BytesRef scratch = new BytesRef(); + for (int i = 0; i < r.maxDoc(); i++) { + assertEquals(TestBinaryDocValuesUpdates.getValue(bdvcf, i, scratch ), TestBinaryDocValuesUpdates.getValue(bdvf, i, scratch)*2); + } + } + + private void verifyDocValues(Directory dir) throws IOException { + DirectoryReader reader = DirectoryReader.open(dir); + for (AtomicReaderContext context : reader.leaves()) { + AtomicReader r = context.reader(); + assertNumericDocValues(r, "ndv1", "ndv1_c"); + assertNumericDocValues(r, "ndv2", "ndv2_c"); + assertBinaryDocValues(r, "bdv1", "bdv1_c"); + assertBinaryDocValues(r, "bdv2", "bdv2_c"); + } + reader.close(); + } + + public void testDocValuesUpdates() throws Exception { + File oldIndexDir = createTempDir("dvupdates"); + TestUtil.unzip(getDataFile(dvUpdatesIndex), oldIndexDir); + Directory dir = newFSDirectory(oldIndexDir); + + verifyDocValues(dir); + + // update fields and verify index + IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + IndexWriter writer = new IndexWriter(dir, conf); + updateNumeric(writer, "1", "ndv1", "ndv1_c", 300L); + updateNumeric(writer, "1", "ndv2", "ndv2_c", 300L); + updateBinary(writer, "1", "bdv1", "bdv1_c", 300L); + updateBinary(writer, "1", "bdv2", "bdv2_c", 300L); + writer.commit(); + verifyDocValues(dir); + + // merge all segments + writer.forceMerge(1); + writer.commit(); + verifyDocValues(dir); + + writer.shutdown(); + dir.close(); + } + } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java index 078088fd3aa..32f17c04173 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java @@ -27,6 +27,7 @@ import org.apache.lucene.document.IntField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; @@ -884,4 +885,155 @@ public class TestTermsEnum extends LuceneTestCase { r.close(); dir.close(); } + + // LUCENE-5667 + public void testCommonPrefixTerms() throws Exception { + Directory d = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), d); + Set terms = new HashSet(); + //String prefix = TestUtil.randomSimpleString(random(), 1, 20); + String prefix = TestUtil.randomRealisticUnicodeString(random(), 1, 20); + int numTerms = atLeast(1000); + if (VERBOSE) { + System.out.println("TEST: " + numTerms + " terms; prefix=" + prefix); + } + while (terms.size() < numTerms) { + //terms.add(prefix + TestUtil.randomSimpleString(random(), 1, 20)); + terms.add(prefix + TestUtil.randomRealisticUnicodeString(random(), 1, 20)); + } + for(String term : terms) { + Document doc = new Document(); + doc.add(newStringField("id", term, Field.Store.YES)); + w.addDocument(doc); + } + IndexReader r = w.getReader(); + if (VERBOSE) { + System.out.println("\nTEST: reader=" + r); + } + + TermsEnum termsEnum = MultiFields.getTerms(r, "id").iterator(null); + DocsEnum docsEnum = null; + PerThreadPKLookup pkLookup = new PerThreadPKLookup(r, "id"); + + int iters = atLeast(numTerms*3); + List termsList = new ArrayList<>(terms); + for(int iter=0;iter leaves = new ArrayList<>(r.leaves()); + + // Larger segments are more likely to have the id, so we sort largest to smallest by numDocs: + Collections.sort(leaves, new Comparator() { + @Override + public int compare(AtomicReaderContext c1, AtomicReaderContext c2) { + return c2.reader().numDocs() - c1.reader().numDocs(); + } + }); + + termsEnums = new TermsEnum[leaves.size()]; + docsEnums = new DocsEnum[leaves.size()]; + liveDocs = new Bits[leaves.size()]; + docBases = new int[leaves.size()]; + int numSegs = 0; + boolean hasDeletions = false; + for(int i=0;i nodeNames = getSortedOverseerNodeNames(zk); diff --git a/solr/core/src/java/org/apache/solr/handler/component/MergeStrategy.java b/solr/core/src/java/org/apache/solr/handler/component/MergeStrategy.java new file mode 100644 index 00000000000..0ff19bd523e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/component/MergeStrategy.java @@ -0,0 +1,76 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.solr.handler.component; + +import org.apache.solr.search.SolrIndexSearcher; + +import java.util.Comparator; +import java.io.IOException; + +/** + * The MergeStrategy class defines custom merge logic for distributed searches. + **/ + + +public interface MergeStrategy { + + /** + * merge defines the merging behaving of results that are collected from the + * shards during a distributed search. + * + **/ + + public void merge(ResponseBuilder rb, ShardRequest sreq); + + /** + * mergesIds must return true if the merge method merges document ids from the shards. + * If it merges other output from the shards it must return false. + * */ + + public boolean mergesIds(); + + + /** + * handlesMergeFields must return true if the MergeStrategy + * implements a custom handleMergeFields(ResponseBuilder rb, SolrIndexSearch searcher) + * */ + + public boolean handlesMergeFields(); + + + /** + * Implement handleMergeFields(ResponseBuilder rb, SolrIndexSearch searcher) if + * your merge strategy needs more complex data then the sort fields provide. + * */ + + public void handleMergeFields(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException; + + /** + * Defines the order that the mergeStrategies are applied. Lower costs are applied first. + * */ + public int getCost(); + + public static final Comparator MERGE_COMP = new Comparator() { + public int compare(Object o1, Object o2) { + MergeStrategy m1 = (MergeStrategy)o1; + MergeStrategy m2 = (MergeStrategy)o2; + return m1.getCost()-m2.getCost(); + } + }; + +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index 3be4f6437e5..ac2689b9d17 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -77,6 +77,7 @@ import org.apache.solr.search.ReturnFields; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrReturnFields; import org.apache.solr.search.SortSpec; +import org.apache.solr.search.RankQuery; import org.apache.solr.search.SyntaxError; import org.apache.solr.search.grouping.CommandHandler; import org.apache.solr.search.grouping.GroupingSpecification; @@ -98,6 +99,8 @@ import org.apache.solr.search.grouping.endresulttransformer.GroupedEndResultTran import org.apache.solr.search.grouping.endresulttransformer.MainEndResultTransformer; import org.apache.solr.search.grouping.endresulttransformer.SimpleEndResultTransformer; import org.apache.solr.util.SolrPluginUtils; +import java.util.Collections; +import java.util.Comparator; /** * TODO! @@ -147,6 +150,17 @@ public class QueryComponent extends SearchComponent // normalize a null query to a query that matches nothing q = new BooleanQuery(); } + + if(q instanceof RankQuery) { + MergeStrategy mergeStrategy = ((RankQuery)q).getMergeStrategy(); + if(mergeStrategy != null) { + rb.addMergeStrategy(mergeStrategy); + if(mergeStrategy.handlesMergeFields()) { + rb.mergeFieldHandler = mergeStrategy; + } + } + } + rb.setQuery( q ); rb.setSortSpec( parser.getSort(true) ); rb.setQparser(parser); @@ -473,7 +487,13 @@ public class QueryComponent extends SearchComponent rb.getNextCursorMark().getSerializedTotem()); } } - doFieldSortValues(rb, searcher); + + if(rb.mergeFieldHandler != null) { + rb.mergeFieldHandler.handleMergeFields(rb, searcher); + } else { + doFieldSortValues(rb, searcher); + } + doPrefetch(rb); } @@ -821,6 +841,22 @@ public class QueryComponent extends SearchComponent private void mergeIds(ResponseBuilder rb, ShardRequest sreq) { + List mergeStrategies = rb.getMergeStrategies(); + if(mergeStrategies != null) { + Collections.sort(mergeStrategies, MergeStrategy.MERGE_COMP); + boolean idsMerged = false; + for(MergeStrategy mergeStrategy : mergeStrategies) { + mergeStrategy.merge(rb, sreq); + if(mergeStrategy.mergesIds()) { + idsMerged = true; + } + } + + if(idsMerged) { + return; //ids were merged above so return. + } + } + SortSpec ss = rb.getSortSpec(); Sort sort = ss.getSort(); diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java index 5fd9fa248b2..c30ac710c5e 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java @@ -40,6 +40,7 @@ import org.apache.solr.search.grouping.distributed.command.QueryCommandResult; import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.ArrayList; import java.util.Map; import java.util.Set; @@ -58,6 +59,7 @@ public class ResponseBuilder public boolean doExpand; public boolean doStats; public boolean doTerms; + public MergeStrategy mergeFieldHandler; private boolean needDocList = false; private boolean needDocSet = false; @@ -74,6 +76,9 @@ public class ResponseBuilder private CursorMark cursorMark; private CursorMark nextCursorMark; + private List mergeStrategies; + + private DocListAndSet results = null; private NamedList debugInfo = null; private RTimer timer = null; @@ -230,7 +235,23 @@ public class ResponseBuilder debugResults = dbg; debugTrack = dbg; } - + + public void addMergeStrategy(MergeStrategy mergeStrategy) { + if(mergeStrategies == null) { + mergeStrategies = new ArrayList(); + } + + mergeStrategies.add(mergeStrategy); + } + + public List getMergeStrategies() { + return this.mergeStrategies; + } + + public void setResponseDocs(SolrDocumentList _responseDocs) { + this._responseDocs = _responseDocs; + } + public boolean isDebugTrack() { return debugTrack; } diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java b/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java index ec1e33f20f2..97b831b3d7f 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java @@ -35,7 +35,7 @@ public class ShardDoc extends FieldDoc { public String shard; public String shardAddress; // TODO - int orderInShard; + public int orderInShard; // the position of this doc within the shard... this can be used // to short-circuit comparisons if the shard is equal, and can // also be used to break ties within the same shard. diff --git a/solr/core/src/java/org/apache/solr/response/XSLTResponseWriter.java b/solr/core/src/java/org/apache/solr/response/XSLTResponseWriter.java index f7f74106c96..4ff55cb9598 100644 --- a/solr/core/src/java/org/apache/solr/response/XSLTResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/XSLTResponseWriter.java @@ -108,9 +108,7 @@ public class XSLTResponseWriter implements QueryResponseWriter { try { t.transform(source, result); } catch(TransformerException te) { - final IOException ioe = new IOException("XSLT transformation error"); - ioe.initCause(te); - throw ioe; + throw new IOException("XSLT transformation error", te); } } diff --git a/solr/core/src/java/org/apache/solr/search/RankQuery.java b/solr/core/src/java/org/apache/solr/search/RankQuery.java new file mode 100644 index 00000000000..da8c00a1387 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/RankQuery.java @@ -0,0 +1,29 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.solr.search; + +import org.apache.lucene.search.TopDocsCollector; +import org.apache.lucene.search.Query; +import org.apache.solr.handler.component.MergeStrategy; + +public abstract class RankQuery extends Query { + + public abstract TopDocsCollector getTopDocsCollector(int len, SolrIndexSearcher.QueryCommand cmd); + public abstract MergeStrategy getMergeStrategy(); + +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index e155290520b..50e82bf352d 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -1499,7 +1499,13 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn * TopDocsCollector to use. */ private TopDocsCollector buildTopDocsCollector(int len, QueryCommand cmd) throws IOException { - + + Query q = cmd.getQuery(); + if(q instanceof RankQuery) { + RankQuery rq = (RankQuery)q; + return rq.getTopDocsCollector(len, cmd); + } + if (null == cmd.getSort()) { assert null == cmd.getCursorMark() : "have cursor but no sort"; return TopScoreDocCollector.create(len, true); diff --git a/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java b/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java index 2263f9fcbc6..07f20724040 100644 --- a/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java +++ b/solr/core/src/java/org/apache/solr/util/SystemIdResolver.java @@ -144,7 +144,7 @@ public final class SystemIdResolver implements EntityResolver, EntityResolver2 { return is; } catch (RuntimeException re) { // unfortunately XInclude fallback only works with IOException, but openResource() never throws that one - throw (IOException) (new IOException(re.getMessage()).initCause(re)); + throw new IOException(re.getMessage(), re); } } else { // resolve all other URIs using the standard resolver diff --git a/solr/core/src/java/org/apache/solr/util/xslt/TransformerProvider.java b/solr/core/src/java/org/apache/solr/util/xslt/TransformerProvider.java index 527d15e65fd..87ab2f93a3c 100644 --- a/solr/core/src/java/org/apache/solr/util/xslt/TransformerProvider.java +++ b/solr/core/src/java/org/apache/solr/util/xslt/TransformerProvider.java @@ -83,9 +83,7 @@ public class TransformerProvider { result = lastTemplates.newTransformer(); } catch(TransformerConfigurationException tce) { log.error(getClass().getName(), "getTransformer", tce); - final IOException ioe = new IOException("newTransformer fails ( " + lastFilename + ")"); - ioe.initCause(tce); - throw ioe; + throw new IOException("newTransformer fails ( " + lastFilename + ")", tce); } return result; @@ -114,9 +112,7 @@ public class TransformerProvider { } } catch (Exception e) { log.error(getClass().getName(), "newTemplates", e); - final IOException ioe = new IOException("Unable to initialize Templates '" + filename + "'"); - ioe.initCause(e); - throw ioe; + throw new IOException("Unable to initialize Templates '" + filename + "'", e); } lastFilename = filename; diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml new file mode 100644 index 00000000000..f1431c61b48 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml @@ -0,0 +1,579 @@ + + + + + + + + + + + + ${solr.data.dir:} + + + + 1000000 + 2000000 + 3000000 + 4000000 + + + ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + + + + + + + + + + + ${solr.ulog.dir:} + + + + ${solr.commitwithin.softcommit:true} + + + + + + + 1024 + + + + + + + + + + + + true + + + + + + 10 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + true + + + + + + dismax + *:* + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + + lowerpunctfilt + + + default + lowerfilt + spellchecker1 + false + + + direct + DirectSolrSpellChecker + lowerfilt + 3 + + + wordbreak + solr.WordBreakSolrSpellChecker + lowerfilt + true + true + 10 + + + multipleFields + lowerfilt1and2 + spellcheckerMultipleFields + false + + + + jarowinkler + lowerfilt + + org.apache.lucene.search.spell.JaroWinklerDistance + spellchecker2 + + + + solr.FileBasedSpellChecker + external + spellings.txt + UTF-8 + spellchecker3 + + + + freq + lowerfilt + spellcheckerFreq + + freq + false + + + fqcn + lowerfilt + spellcheckerFQCN + org.apache.solr.spelling.SampleComparator + false + + + perDict + org.apache.solr.handler.component.DummyCustomParamSpellChecker + lowerfilt + + + + + + + + termsComp + + + + + + + + + false + + false + + 1 + + + spellcheck + + + + + direct + false + false + 1 + + + spellcheck + + + + + default + wordbreak + 20 + + + spellcheck + + + + + direct + wordbreak + 20 + + + spellcheck + + + + + dismax + lowerfilt1^1 + + + spellcheck + + + + + + + + + + + + + + + tvComponent + + + + + + string + elevate.xml + + + + + + explicit + + + elevate + + + + + + + + + + + + + 100 + + + + + + 70 + + + + + + + ]]> + ]]> + + + + + + + + + + + + + 10 + .,!? + + + + + + WORD + en + US + + + + + + + + + + max-age=30, public + + + + + + + explicit + true + + + + + solr + solrconfig.xml schema.xml admin-extra.html + + + + prefix-${solr.test.sys.prop2}-suffix + + + + + + false + true + v_t,t_field + org.apache.solr.update.processor.TextProfileSignature + + + + + + false + false + id + + org.apache.solr.update.processor.Lookup3Signature + + + + + + + true + non_indexed_signature_sS + false + v_t,t_field + org.apache.solr.update.processor.TextProfileSignature + + + + + + + uniq + uniq2 + uniq3 + + + + + + + + + regex_dup_A_s + x + x_x + + + + regex_dup_B_s + x + x_x + + + + + + + + regex_dup_A_s + x + x_x + + + regex_dup_B_s + x + x_x + + + + + + diff --git a/solr/core/src/test/org/apache/solr/search/MergeStrategyTest.java b/solr/core/src/test/org/apache/solr/search/MergeStrategyTest.java new file mode 100644 index 00000000000..68b8c9e8a7d --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/MergeStrategyTest.java @@ -0,0 +1,181 @@ +package org.apache.solr.search; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.solr.BaseDistributedSearchTestCase; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.handler.component.MergeStrategy; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.handler.component.ShardRequest; +import org.junit.BeforeClass; + +import java.util.Arrays; + +/** + * Test for QueryComponent's distributed querying + * + * @see org.apache.solr.handler.component.QueryComponent + */ +public class MergeStrategyTest extends BaseDistributedSearchTestCase { + + public MergeStrategyTest() { + fixShardCount = true; + shardCount = 3; + stress = 0; + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + initCore("solrconfig-plugcollector.xml", "schema15.xml"); + } + + @Override + public void doTest() throws Exception { + del("*:*"); + + index_specific(0,"id","1", "sort_i", "5"); + index_specific(0,"id","2", "sort_i", "50"); + index_specific(1,"id","5", "sort_i", "4"); + index_specific(1,"id","6", "sort_i", "10"); + index_specific(0,"id","7", "sort_i", "1"); + index_specific(1,"id","8", "sort_i", "2"); + index_specific(2,"id","9", "sort_i", "1000"); + index_specific(2,"id","10", "sort_i", "1500"); + index_specific(2,"id","11", "sort_i", "1300"); + index_specific(1,"id","12", "sort_i", "15"); + index_specific(1,"id","13", "sort_i", "16"); + + commit(); + + handle.put("explain", SKIPVAL); + handle.put("QTime", SKIPVAL); + handle.put("timestamp", SKIPVAL); + handle.put("score", SKIPVAL); + handle.put("wt", SKIP); + handle.put("distrib", SKIP); + handle.put("shards.qt", SKIP); + handle.put("shards", SKIP); + handle.put("q", SKIP); + handle.put("maxScore", SKIPVAL); + handle.put("_version_", SKIP); + + //Test mergeStrategy that uses score + query("q", "{!rank q=$qq}", "qq", "*:*", "rows","12", "sort", "sort_i asc", "fl","*,score"); + + //Test without mergeStrategy + query("q", "*:*", "rows","12", "sort", "sort_i asc"); + + //Test mergeStrategy1 that uses a sort field. + query("q", "{!rank mergeStrategy=1 q=$qq}", "qq", "*:*", "rows","12", "sort", "sort_i asc"); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.add("qq", "*:*"); + params.add("rows", "12"); + params.add("q", "{!rank q=$qq}"); + params.add("sort", "sort_i asc"); + params.add("fl","*,score"); + setDistributedParams(params); + QueryResponse rsp = queryServer(params); + assertOrder(rsp,"10","11","9","2","13","12","6","1","5","8","7"); + + params = new ModifiableSolrParams(); + params.add("q", "*:*"); + params.add("rows", "12"); + params.add("sort", "sort_i asc"); + params.add("fl","*,score"); + setDistributedParams(params); + rsp = queryServer(params); + assertOrder(rsp,"7","8","5","1","6","12","13","2","9","11","10"); + + MergeStrategy m1 = new MergeStrategy() { + @Override + public void merge(ResponseBuilder rb, ShardRequest sreq) { + } + + public boolean mergesIds() { + return true; + } + + public boolean handlesMergeFields() { return false;} + public void handleMergeFields(ResponseBuilder rb, SolrIndexSearcher searcher) {} + + @Override + public int getCost() { + return 1; + } + }; + + MergeStrategy m2 = new MergeStrategy() { + @Override + public void merge(ResponseBuilder rb, ShardRequest sreq) { + } + + public boolean mergesIds() { + return true; + } + + public boolean handlesMergeFields() { return false;} + public void handleMergeFields(ResponseBuilder rb, SolrIndexSearcher searcher) {} + + @Override + public int getCost() { + return 100; + } + }; + + MergeStrategy m3 = new MergeStrategy() { + @Override + public void merge(ResponseBuilder rb, ShardRequest sreq) { + } + + public boolean mergesIds() { + return false; + } + + public boolean handlesMergeFields() { return false;} + public void handleMergeFields(ResponseBuilder rb, SolrIndexSearcher searcher) {} + + @Override + public int getCost() { + return 50; + } + }; + + MergeStrategy[] merges = {m1,m2,m3}; + Arrays.sort(merges, MergeStrategy.MERGE_COMP); + assert(merges[0].getCost() == 1); + assert(merges[1].getCost() == 50); + assert(merges[2].getCost() == 100); + } + + private void assertOrder(QueryResponse rsp, String ... docs) throws Exception { + SolrDocumentList list = rsp.getResults(); + for(int i=0; i uniqueDoc = new HashMap<>(); + + + NamedList shardInfo = null; + if(rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) { + shardInfo = new SimpleOrderedMap<>(); + rb.rsp.getValues().add(ShardParams.SHARDS_INFO,shardInfo); + } + + IndexSchema schema = rb.req.getSchema(); + SchemaField uniqueKeyField = schema.getUniqueKeyField(); + + long numFound = 0; + Float maxScore=null; + boolean partialResults = false; + List shardDocs = new ArrayList(); + + for (ShardResponse srsp : sreq.responses) { + SolrDocumentList docs = null; + + if(shardInfo!=null) { + SimpleOrderedMap nl = new SimpleOrderedMap<>(); + + if (srsp.getException() != null) { + Throwable t = srsp.getException(); + if(t instanceof SolrServerException) { + t = ((SolrServerException)t).getCause(); + } + nl.add("error", t.toString() ); + StringWriter trace = new StringWriter(); + t.printStackTrace(new PrintWriter(trace)); + nl.add("trace", trace.toString() ); + if (srsp.getShardAddress() != null) { + nl.add("shardAddress", srsp.getShardAddress()); + } + } + else { + docs = (SolrDocumentList)srsp.getSolrResponse().getResponse().get("response"); + nl.add("numFound", docs.getNumFound()); + nl.add("maxScore", docs.getMaxScore()); + nl.add("shardAddress", srsp.getShardAddress()); + } + if(srsp.getSolrResponse()!=null) { + nl.add("time", srsp.getSolrResponse().getElapsedTime()); + } + + shardInfo.add(srsp.getShard(), nl); + } + // now that we've added the shard info, let's only proceed if we have no error. + if (srsp.getException() != null) { + partialResults = true; + continue; + } + + if (docs == null) { // could have been initialized in the shards info block above + docs = (SolrDocumentList)srsp.getSolrResponse().getResponse().get("response"); + } + + NamedList responseHeader = (NamedList)srsp.getSolrResponse().getResponse().get("responseHeader"); + if (responseHeader != null && Boolean.TRUE.equals(responseHeader.get("partialResults"))) { + partialResults = true; + } + + // calculate global maxScore and numDocsFound + if (docs.getMaxScore() != null) { + maxScore = maxScore==null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore()); + } + numFound += docs.getNumFound(); + + + for (int i=0; i= 0) { + // TODO: remove previous from priority queue + // continue; + // } + } + + ShardDoc shardDoc = new ShardDoc(); + shardDoc.id = id; + shardDoc.shard = srsp.getShard(); + shardDoc.orderInShard = i; + Object scoreObj = doc.getFieldValue("score"); + if (scoreObj != null) { + if (scoreObj instanceof String) { + shardDoc.score = Float.parseFloat((String)scoreObj); + } else { + shardDoc.score = (Float)scoreObj; + } + } + shardDocs.add(shardDoc); + } // end for-each-doc-in-response + } // end for-each-response + + Collections.sort(shardDocs, new Comparator() { + @Override + public int compare(ShardDoc o1, ShardDoc o2) { + if(o1.score < o2.score) { + return 1; + } else if (o1.score > o2.score) { + return -1; + } else { + return 0; //To change body of implemented methods use File | Settings | File Templates. + } + } + }); + + int resultSize = shardDocs.size(); + + Map resultIds = new HashMap<>(); + for (int i=0; i sortVals = new NamedList<>(); // order is important for the sort fields + IndexReaderContext topReaderContext = searcher.getTopReaderContext(); + List leaves = topReaderContext.leaves(); + AtomicReaderContext currentLeaf = null; + if (leaves.size()==1) { + // if there is a single segment, use that subReader and avoid looking up each time + currentLeaf = leaves.get(0); + leaves=null; + } + + DocList docList = rb.getResults().docList; + + // sort ids from lowest to highest so we can access them in order + int nDocs = docList.size(); + final long[] sortedIds = new long[nDocs]; + final float[] scores = new float[nDocs]; // doc scores, parallel to sortedIds + DocList docs = rb.getResults().docList; + DocIterator it = docs.iterator(); + for (int i=0; i schemaFields = sortSpec.getSchemaFields(); + + for (int fld = 0; fld < schemaFields.size(); fld++) { + SchemaField schemaField = schemaFields.get(fld); + FieldType ft = null == schemaField? null : schemaField.getType(); + SortField sortField = sortFields[fld]; + + SortField.Type type = sortField.getType(); + // :TODO: would be simpler to always serialize every position of SortField[] + if (type==SortField.Type.SCORE || type==SortField.Type.DOC) continue; + + FieldComparator comparator = null; + Object[] vals = new Object[nDocs]; + + int lastIdx = -1; + int idx = 0; + + for (int i = 0; i < sortedIds.length; ++i) { + long idAndPos = sortedIds[i]; + float score = scores[i]; + int doc = (int)(idAndPos >>> 32); + int position = (int)idAndPos; + + if (leaves != null) { + idx = ReaderUtil.subIndex(doc, leaves); + currentLeaf = leaves.get(idx); + if (idx != lastIdx) { + // we switched segments. invalidate comparator. + comparator = null; + } + } + + if (comparator == null) { + comparator = sortField.getComparator(1,0); + comparator = comparator.setNextReader(currentLeaf); + } + + doc -= currentLeaf.docBase; // adjust for what segment this is in + comparator.setScorer(new FakeScorer(doc, score)); + comparator.copy(0, doc); + Object val = comparator.value(0); + if (null != ft) val = ft.marshalSortValue(val); + vals[position] = val; + } + + sortVals.add(sortField.getField(), vals); + } + + rsp.add("merge_values", sortVals); + } + } + + private class FakeScorer extends Scorer { + final int docid; + final float score; + + FakeScorer(int docid, float score) { + super(null); + this.docid = docid; + this.score = score; + } + + @Override + public int docID() { + return docid; + } + + @Override + public float score() throws IOException { + return score; + } + + @Override + public int freq() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + return 1; + } + + @Override + public Weight getWeight() { + throw new UnsupportedOperationException(); + } + + @Override + public Collection getChildren() { + throw new UnsupportedOperationException(); + } + } + + public void merge(ResponseBuilder rb, ShardRequest sreq) { + + // id to shard mapping, to eliminate any accidental dups + HashMap uniqueDoc = new HashMap<>(); + + + NamedList shardInfo = null; + if(rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) { + shardInfo = new SimpleOrderedMap<>(); + rb.rsp.getValues().add(ShardParams.SHARDS_INFO,shardInfo); + } + + IndexSchema schema = rb.req.getSchema(); + SchemaField uniqueKeyField = schema.getUniqueKeyField(); + + long numFound = 0; + Float maxScore=null; + boolean partialResults = false; + List shardDocs = new ArrayList(); + + for (ShardResponse srsp : sreq.responses) { + SolrDocumentList docs = null; + + if(shardInfo!=null) { + SimpleOrderedMap nl = new SimpleOrderedMap<>(); + + if (srsp.getException() != null) { + Throwable t = srsp.getException(); + if(t instanceof SolrServerException) { + t = ((SolrServerException)t).getCause(); + } + nl.add("error", t.toString() ); + StringWriter trace = new StringWriter(); + t.printStackTrace(new PrintWriter(trace)); + nl.add("trace", trace.toString() ); + if (srsp.getShardAddress() != null) { + nl.add("shardAddress", srsp.getShardAddress()); + } + } + else { + docs = (SolrDocumentList)srsp.getSolrResponse().getResponse().get("response"); + nl.add("numFound", docs.getNumFound()); + nl.add("maxScore", docs.getMaxScore()); + nl.add("shardAddress", srsp.getShardAddress()); + } + if(srsp.getSolrResponse()!=null) { + nl.add("time", srsp.getSolrResponse().getElapsedTime()); + } + + shardInfo.add(srsp.getShard(), nl); + } + // now that we've added the shard info, let's only proceed if we have no error. + if (srsp.getException() != null) { + partialResults = true; + continue; + } + + if (docs == null) { // could have been initialized in the shards info block above + docs = (SolrDocumentList)srsp.getSolrResponse().getResponse().get("response"); + } + + NamedList responseHeader = (NamedList)srsp.getSolrResponse().getResponse().get("responseHeader"); + if (responseHeader != null && Boolean.TRUE.equals(responseHeader.get("partialResults"))) { + partialResults = true; + } + + // calculate global maxScore and numDocsFound + if (docs.getMaxScore() != null) { + maxScore = maxScore==null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore()); + } + numFound += docs.getNumFound(); + + SortSpec ss = rb.getSortSpec(); + Sort sort = ss.getSort(); + + NamedList sortFieldValues = (NamedList)(srsp.getSolrResponse().getResponse().get("merge_values")); + NamedList unmarshalledSortFieldValues = unmarshalSortValues(ss, sortFieldValues, schema); + List lst = (List)unmarshalledSortFieldValues.getVal(0); + + for (int i=0; i= 0) { + // TODO: remove previous from priority queue + // continue; + // } + } + + ShardDoc shardDoc = new ShardDoc(); + shardDoc.id = id; + shardDoc.shard = srsp.getShard(); + shardDoc.orderInShard = i; + Object scoreObj = lst.get(i); + if (scoreObj != null) { + shardDoc.score = ((Integer)scoreObj).floatValue(); + } + shardDocs.add(shardDoc); + } // end for-each-doc-in-response + } // end for-each-response + + Collections.sort(shardDocs, new Comparator() { + @Override + public int compare(ShardDoc o1, ShardDoc o2) { + if(o1.score < o2.score) { + return 1; + } else if (o1.score > o2.score) { + return -1; + } else { + return 0; //To change body of implemented methods use File | Settings | File Templates. + } + } + }); + + int resultSize = shardDocs.size(); + + Map resultIds = new HashMap<>(); + for (int i=0; i schemaFields = sortSpec.getSchemaFields(); + SortField[] sortFields = sortSpec.getSort().getSort(); + + int marshalledFieldNum = 0; + for (int sortFieldNum = 0; sortFieldNum < sortFields.length; sortFieldNum++) { + final SortField sortField = sortFields[sortFieldNum]; + final SortField.Type type = sortField.getType(); + + // :TODO: would be simpler to always serialize every position of SortField[] + if (type==SortField.Type.SCORE || type==SortField.Type.DOC) continue; + + final String sortFieldName = sortField.getField(); + final String valueFieldName = sortFieldValues.getName(marshalledFieldNum); + assert sortFieldName.equals(valueFieldName) + : "sortFieldValues name key does not match expected SortField.getField"; + + List sortVals = (List)sortFieldValues.getVal(marshalledFieldNum); + + final SchemaField schemaField = schemaFields.get(sortFieldNum); + if (null == schemaField) { + unmarshalledSortValsPerField.add(sortField.getField(), sortVals); + } else { + FieldType fieldType = schemaField.getType(); + List unmarshalledSortVals = new ArrayList(); + for (Object sortVal : sortVals) { + unmarshalledSortVals.add(fieldType.unmarshalSortValue(sortVal)); + } + unmarshalledSortValsPerField.add(sortField.getField(), unmarshalledSortVals); + } + marshalledFieldNum++; + } + return unmarshalledSortValsPerField; + } + } + + + class TestCollector extends TopDocsCollector { + + private List list = new ArrayList(); + private NumericDocValues values; + private int base; + + public TestCollector(PriorityQueue pq) { + super(pq); + } + + public boolean acceptsDocsOutOfOrder() { + return false; + } + + public void doSetNextReader(AtomicReaderContext context) throws IOException { + values = DocValues.getNumeric(context.reader(), "sort_i"); + base = context.docBase; + } + + public void collect(int doc) { + list.add(new ScoreDoc(doc+base, (float)values.get(doc))); + } + + public int topDocsSize() { + return list.size(); + } + + public TopDocs topDocs() { + Collections.sort(list, new Comparator() { + public int compare(Object o1, Object o2) { + ScoreDoc s1 = (ScoreDoc) o1; + ScoreDoc s2 = (ScoreDoc) o2; + if (s1.score == s2.score) { + return 0; + } else if (s1.score < s2.score) { + return 1; + } else { + return -1; + } + } + }); + ScoreDoc[] scoreDocs = list.toArray(new ScoreDoc[list.size()]); + return new TopDocs(list.size(), scoreDocs, 0.0f); + } + + public TopDocs topDocs(int start, int len) { + return topDocs(); + } + + public int getTotalHits() { + return list.size(); + } + } + + class TestCollector1 extends TopDocsCollector { + + private List list = new ArrayList(); + private int base; + private Scorer scorer; + + public TestCollector1(PriorityQueue pq) { + super(pq); + } + + public boolean acceptsDocsOutOfOrder() { + return false; + } + + public void doSetNextReader(AtomicReaderContext context) throws IOException { + base = context.docBase; + } + + public void setScorer(Scorer scorer) { + this.scorer = scorer; + } + + public void collect(int doc) throws IOException { + list.add(new ScoreDoc(doc+base, scorer.score())); + } + + public int topDocsSize() { + return list.size(); + } + + public TopDocs topDocs() { + Collections.sort(list, new Comparator() { + public int compare(Object o1, Object o2) { + ScoreDoc s1 = (ScoreDoc) o1; + ScoreDoc s2 = (ScoreDoc) o2; + if (s1.score == s2.score) { + return 0; + } else if (s1.score > s2.score) { + return 1; + } else { + return -1; + } + } + }); + ScoreDoc[] scoreDocs = list.toArray(new ScoreDoc[list.size()]); + return new TopDocs(list.size(), scoreDocs, 0.0f); + } + + public TopDocs topDocs(int start, int len) { + return topDocs(); + } + + public int getTotalHits() { + return list.size(); + } + } + + + + +} diff --git a/solr/licenses/icu4j-52.1.jar.sha1 b/solr/licenses/icu4j-52.1.jar.sha1 deleted file mode 100644 index d3551e8380b..00000000000 --- a/solr/licenses/icu4j-52.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7dbc327670673acd14b487d120f05747d712c1c0 diff --git a/solr/licenses/icu4j-53.1.jar.sha1 b/solr/licenses/icu4j-53.1.jar.sha1 new file mode 100644 index 00000000000..ac60dac2311 --- /dev/null +++ b/solr/licenses/icu4j-53.1.jar.sha1 @@ -0,0 +1 @@ +786d9055d4ca8c1aab4a7d4ac8283f973fd7e41f