From af05ac30c4f543724972708be281b24f3fa5cae5 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 15 Feb 2013 20:58:54 +0000 Subject: [PATCH 01/13] LUCENE-4779: add missing values tests git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446762 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/lucene/search/TestSort.java | 64 --- .../org/apache/lucene/search/TestSort2.java | 371 +++++++++++++++++- 2 files changed, 370 insertions(+), 65 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSort.java b/lucene/core/src/test/org/apache/lucene/search/TestSort.java index c39686cd263..57bff3ffe9d 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSort.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSort.java @@ -69,12 +69,10 @@ public class TestSort extends LuceneTestCase { private IndexSearcher searchX; private IndexSearcher searchY; private Query queryX; - private Query queryY; private Query queryA; private Query queryE; private Query queryF; private Query queryG; - private Query queryM; private Sort sort; @BeforeClass @@ -355,12 +353,10 @@ public class TestSort extends LuceneTestCase { searchX = getXIndex(); searchY = getYIndex(); queryX = new TermQuery(new Term("contents", "x")); - queryY = new TermQuery(new Term("contents", "y")); queryA = new TermQuery(new Term("contents", "a")); queryE = new TermQuery(new Term("contents", "e")); queryF = new TermQuery(new Term("contents", "f")); queryG = new TermQuery(new Term("contents", "g")); - queryM = new TermQuery(new Term("contents", "m")); sort = new Sort(); } @@ -391,66 +387,6 @@ public class TestSort extends LuceneTestCase { return SortField.Type.STRING_VAL; } } - - private static class SortMissingLastTestHelper { - final SortField sortField; - final Object min; - final Object max; - - SortMissingLastTestHelper(SortField sortField, Object min, Object max) { - this.sortField = sortField; - this.min = min; - this.max = max; - } - } - - // test sorts where the type of field is specified - public void testSortMissingLast() throws Exception { - - @SuppressWarnings("boxing") - SortMissingLastTestHelper[] ascendTesters = new SortMissingLastTestHelper[] { - new SortMissingLastTestHelper( new SortField( "byte", SortField.Type.BYTE ), Byte.MIN_VALUE, Byte.MAX_VALUE ), - new SortMissingLastTestHelper( new SortField( "short", SortField.Type.SHORT ), Short.MIN_VALUE, Short.MAX_VALUE ), - new SortMissingLastTestHelper( new SortField( "int", SortField.Type.INT ), Integer.MIN_VALUE, Integer.MAX_VALUE ), - new SortMissingLastTestHelper( new SortField( "long", SortField.Type.LONG ), Long.MIN_VALUE, Long.MAX_VALUE ), - new SortMissingLastTestHelper( new SortField( "float", SortField.Type.FLOAT ), Float.MIN_VALUE, Float.MAX_VALUE ), - new SortMissingLastTestHelper( new SortField( "double", SortField.Type.DOUBLE ), Double.MIN_VALUE, Double.MAX_VALUE ), - }; - - @SuppressWarnings("boxing") - SortMissingLastTestHelper[] descendTesters = new SortMissingLastTestHelper[] { - new SortMissingLastTestHelper( new SortField( "byte", SortField.Type.BYTE, true ), Byte.MIN_VALUE, Byte.MAX_VALUE ), - new SortMissingLastTestHelper( new SortField( "short", SortField.Type.SHORT, true ), Short.MIN_VALUE, Short.MAX_VALUE ), - new SortMissingLastTestHelper( new SortField( "int", SortField.Type.INT, true ), Integer.MIN_VALUE, Integer.MAX_VALUE ), - new SortMissingLastTestHelper( new SortField( "long", SortField.Type.LONG, true ), Long.MIN_VALUE, Long.MAX_VALUE ), - new SortMissingLastTestHelper( new SortField( "float", SortField.Type.FLOAT, true ), Float.MIN_VALUE, Float.MAX_VALUE ), - new SortMissingLastTestHelper( new SortField( "double", SortField.Type.DOUBLE, true ), Double.MIN_VALUE, Double.MAX_VALUE ), - }; - - // Default order: ascending - for(SortMissingLastTestHelper t : ascendTesters) { - sort.setSort(t.sortField, SortField.FIELD_DOC); - assertMatches("sortField:"+t.sortField, full, queryM, sort, "adbc"); - - sort.setSort(t.sortField.setMissingValue(t.max), SortField.FIELD_DOC); - assertMatches("sortField:"+t.sortField, full, queryM, sort, "bcad"); - - sort.setSort(t.sortField.setMissingValue(t.min), SortField.FIELD_DOC); - assertMatches("sortField:"+t.sortField, full, queryM, sort, "adbc"); - } - - // Reverse order: descending (Note: Order for un-valued documents remains the same due to tie breaker: a,d) - for(SortMissingLastTestHelper t : descendTesters) { - sort.setSort(t.sortField, SortField.FIELD_DOC); - assertMatches("sortField:"+t.sortField, full, queryM, sort, "cbad"); - - sort.setSort(t.sortField.setMissingValue( t.max ), SortField.FIELD_DOC); - assertMatches("sortField:"+t.sortField, full, queryM, sort, "adcb"); - - sort.setSort(t.sortField.setMissingValue( t.min ), SortField.FIELD_DOC); - assertMatches("sortField:"+t.sortField, full, queryM, sort, "cbad"); - } - } /** * Test String sorting: small queue to many matches, multi field sort, reverse sort diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSort2.java b/lucene/core/src/test/org/apache/lucene/search/TestSort2.java index b26f9029dee..83ea3c33298 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSort2.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSort2.java @@ -301,6 +301,66 @@ public class TestSort2 extends LuceneTestCase { dir.close(); } + /** Tests sorting on type byte with a missing value */ + public void testByteMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.BYTE)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null value is treated as a 0 + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type byte, specifying the missing value should be treated as Byte.MAX_VALUE */ + public void testByteMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + SortField sortField = new SortField("value", SortField.Type.BYTE); + sortField.setMissingValue(Byte.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null value is treated Byte.MAX_VALUE + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + /** Tests sorting on type byte in reverse */ public void testByteReverse() throws IOException { Directory dir = newDirectory(); @@ -361,6 +421,66 @@ public class TestSort2 extends LuceneTestCase { dir.close(); } + /** Tests sorting on type short with a missing value */ + public void testShortMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.SHORT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as a 0 + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type short, specifying the missing value should be treated as Short.MAX_VALUE */ + public void testShortMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + SortField sortField = new SortField("value", SortField.Type.SHORT); + sortField.setMissingValue(Short.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as Short.MAX_VALUE + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + /** Tests sorting on type short in reverse */ public void testShortReverse() throws IOException { Directory dir = newDirectory(); @@ -421,6 +541,66 @@ public class TestSort2 extends LuceneTestCase { dir.close(); } + /** Tests sorting on type int with a missing value */ + public void testIntMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.INT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as a 0 + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type int, specifying the missing value should be treated as Integer.MAX_VALUE */ + public void testIntMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + SortField sortField = new SortField("value", SortField.Type.INT); + sortField.setMissingValue(Integer.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as a Integer.MAX_VALUE + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + /** Tests sorting on type int in reverse */ public void testIntReverse() throws IOException { Directory dir = newDirectory(); @@ -481,6 +661,66 @@ public class TestSort2 extends LuceneTestCase { dir.close(); } + /** Tests sorting on type long with a missing value */ + public void testLongMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.LONG)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as 0 + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type long, specifying the missing value should be treated as Long.MAX_VALUE */ + public void testLongMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + SortField sortField = new SortField("value", SortField.Type.LONG); + sortField.setMissingValue(Long.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as Long.MAX_VALUE + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + /** Tests sorting on type long in reverse */ public void testLongReverse() throws IOException { Directory dir = newDirectory(); @@ -541,6 +781,66 @@ public class TestSort2 extends LuceneTestCase { dir.close(); } + /** Tests sorting on type float with a missing value */ + public void testFloatMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1.3", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.FLOAT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as 0 + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type float, specifying the missing value should be treated as Float.MAX_VALUE */ + public void testFloatMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1.3", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + SortField sortField = new SortField("value", SortField.Type.FLOAT); + sortField.setMissingValue(Float.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as Float.MAX_VALUE + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + /** Tests sorting on type float in reverse */ public void testFloatReverse() throws IOException { Directory dir = newDirectory(); @@ -605,6 +905,74 @@ public class TestSort2 extends LuceneTestCase { dir.close(); } + /** Tests sorting on type double with a missing value */ + public void testDoubleMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1.3", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.DOUBLE)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(4, td.totalHits); + // null treated as a 0 + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[2].doc).get("value")); + assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[3].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type double, specifying the missing value should be treated as Double.MAX_VALUE */ + public void testDoubleMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1.3", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + SortField sortField = new SortField("value", SortField.Type.DOUBLE); + sortField.setMissingValue(Double.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(4, td.totalHits); + // null treated as Double.MAX_VALUE + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[2].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[3].doc).get("value")); + + ir.close(); + dir.close(); + } + /** Tests sorting on type double in reverse */ public void testDoubleReverse() throws IOException { Directory dir = newDirectory(); @@ -664,7 +1032,8 @@ public class TestSort2 extends LuceneTestCase { dir.close(); } - public void testLUCENE2142() throws IOException { + /** test that we don't throw exception on multi-valued field (LUCENE-2142) */ + public void testMultiValuedField() throws IOException { Directory indexStore = newDirectory(); IndexWriter writer = new IndexWriter(indexStore, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); From 3d0bd71f49b89d93eff6cbb5b27855c7f54ba992 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 15 Feb 2013 21:17:36 +0000 Subject: [PATCH 02/13] LUCENE-4779: add remaining tests git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446776 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/lucene/search/TestSort2.java | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSort2.java b/lucene/core/src/test/org/apache/lucene/search/TestSort2.java index 83ea3c33298..52c22bc8cfd 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSort2.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSort2.java @@ -82,6 +82,35 @@ public class TestSort2 extends LuceneTestCase { dir.close(); } + /** Tests sorting on type string with a missing value */ + public void testStringMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "foo", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "bar", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.STRING)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null comes first + assertNull(searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("foo", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + /** Tests reverse sorting on type string */ public void testStringReverse() throws IOException { Directory dir = newDirectory(); @@ -134,6 +163,35 @@ public class TestSort2 extends LuceneTestCase { dir.close(); } + /** Tests sorting on type string_val with a missing value */ + public void testStringValMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "foo", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "bar", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.STRING_VAL)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null comes first + assertNull(searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("foo", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + /** Tests reverse sorting on type string_val */ public void testStringValReverse() throws IOException { Directory dir = newDirectory(); @@ -1383,4 +1441,79 @@ public class TestSort2 extends LuceneTestCase { ir.close(); dir.close(); } + + /** Tests sorting a single document */ + public void testSortOneDocument() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "foo", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.STRING)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(1, td.totalHits); + assertEquals("foo", searcher.doc(td.scoreDocs[0].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting a single document with scores */ + public void testSortOneDocumentWithScores() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "foo", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.STRING)); + + TopDocs expected = searcher.search(new TermQuery(new Term("value", "foo")), 10); + assertEquals(1, expected.totalHits); + TopDocs actual = searcher.search(new TermQuery(new Term("value", "foo")), null, 10, sort, true, true); + + assertEquals(expected.totalHits, actual.totalHits); + assertEquals(expected.scoreDocs[0].score, actual.scoreDocs[0].score, 0F); + + ir.close(); + dir.close(); + } + + /** Tests sorting with two fields */ + public void testSortTwoFields() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("tievalue", "tied", Field.Store.NO)); + doc.add(newStringField("value", "foo", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("tievalue", "tied", Field.Store.NO)); + doc.add(newStringField("value", "bar", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + // tievalue, then value + Sort sort = new Sort(new SortField("tievalue", SortField.Type.STRING), + new SortField("value", SortField.Type.STRING)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(2, td.totalHits); + // 'bar' comes before 'foo' + assertEquals("bar", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("foo", searcher.doc(td.scoreDocs[1].doc).get("value")); + + ir.close(); + dir.close(); + } } From 1a12ff7b5ccac90ecaa01b25af44faa2a7253881 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 15 Feb 2013 21:19:11 +0000 Subject: [PATCH 03/13] LUCENE-4779: replace old TestSort git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446777 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/lucene/search/TestSort.java | 2173 +++++++++++------ .../org/apache/lucene/search/TestSort2.java | 1519 ------------ 2 files changed, 1439 insertions(+), 2253 deletions(-) delete mode 100644 lucene/core/src/test/org/apache/lucene/search/TestSort2.java diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSort.java b/lucene/core/src/test/org/apache/lucene/search/TestSort.java index 57bff3ffe9d..7345aeee587 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSort.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSort.java @@ -18,495 +18,1188 @@ package org.apache.lucene.search; */ import java.io.IOException; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.FloatDocValuesField; -import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.FieldInfo.DocValuesType; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.StorableField; -import org.apache.lucene.index.StoredDocument; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.DocIdBitSet; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.NamedThreadFactory; -import org.apache.lucene.util._TestUtil; -import org.junit.BeforeClass; -/** - * Unit tests for sorting code. - * - *

Created: Feb 17, 2004 4:55:10 PM - * - * @since lucene 1.4 + +/* + * Very simple tests of sorting. + * + * THE RULES: + * 1. keywords like 'abstract' and 'static' should not appear in this file. + * 2. each test method should be self-contained and understandable. + * 3. no test methods should share code with other test methods. + * 4. no testing of things unrelated to sorting. + * 5. no tracers. + * 6. keyword 'class' should appear only once in this file, here ---- + * | + * ----------------------------------------------------------- + * | + * \./ */ - public class TestSort extends LuceneTestCase { - private static int NUM_STRINGS; - private IndexSearcher full; - private IndexSearcher searchX; - private IndexSearcher searchY; - private Query queryX; - private Query queryA; - private Query queryE; - private Query queryF; - private Query queryG; - private Sort sort; - @BeforeClass - public static void beforeClass() { - NUM_STRINGS = atLeast(500); - } - - // document data: - // the tracer field is used to determine which document was hit - // the contents field is used to search and sort by relevance - // the int field to sort by int - // the float field to sort by float - // the string field to sort by string - // the i18n field includes accented characters for testing locale-specific sorting - private String[][] data = new String[][] { - // tracer contents int float string custom i18n long double, short, byte, 'custom parser encoding' - { "A", "x a", "5", "4f", "c", "A-3", "p\u00EAche", "10", "-4.0", "3", "126", "J"},//A, x - { "B", "y a", "5", "3.4028235E38", "i", "B-10", "HAT", "1000000000", "40.0", "24", "1", "I"},//B, y - { "C", "x a b c", "2147483647", "1.0", "j", "A-2", "p\u00E9ch\u00E9", "99999999","40.00002343", "125", "15", "H"},//C, x - { "D", "y a b c", "-1", "0.0f", "a", "C-0", "HUT", String.valueOf(Long.MAX_VALUE),String.valueOf(Double.MIN_VALUE), String.valueOf(Short.MIN_VALUE), String.valueOf(Byte.MIN_VALUE), "G"},//D, y - { "E", "x a b c d", "5", "2f", "h", "B-8", "peach", String.valueOf(Long.MIN_VALUE),String.valueOf(Double.MAX_VALUE), String.valueOf(Short.MAX_VALUE), String.valueOf(Byte.MAX_VALUE), "F"},//E,x - { "F", "y a b c d", "2", "3.14159f", "g", "B-1", "H\u00C5T", "-44", "343.034435444", "-3", "0", "E"},//F,y - { "G", "x a b c d", "3", "-1.0", "f", "C-100", "sin", "323254543543", "4.043544", "5", "100", "D"},//G,x - { "H", "y a b c d", "0", "1.4E-45", "e", "C-88", "H\u00D8T", "1023423423005","4.043545", "10", "-50", "C"},//H,y - { "I", "x a b c d e f", "-2147483648", "1.0e+0", "d", "A-10", "s\u00EDn", "332422459999", "4.043546", "-340", "51", "B"},//I,x - { "J", "y a b c d e f", "4", ".5", "b", "C-7", "HOT", "34334543543", "4.0000220343", "300", "2", "A"},//J,y - { "W", "g", "1", null, null, null, null, null, null, null, null, null}, - { "X", "g", "1", "0.1", null, null, null, null, null, null, null, null}, - { "Y", "g", "1", "0.2", null, null, null, null, null, null, null, null}, - { "Z", "f g", null, null, null, null, null, null, null, null, null, null}, - - // Sort Missing first/last - { "a", "m", null, null, null, null, null, null, null, null, null, null}, - { "b", "m", "4", "4.0", "4", null, null, "4", "4", "4", "4", null}, - { "c", "m", "5", "5.0", "5", null, null, "5", "5", "5", "5", null}, - { "d", "m", null, null, null, null, null, null, null, null, null, null} - }; - - // create an index of all the documents, or just the x, or just the y documents - private IndexSearcher getIndex(boolean even, boolean odd) - throws IOException { - Directory indexStore = newDirectory(); - dirs.add(indexStore); - RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); - - final DocValuesType stringDVType; - if (dvStringSorted) { - // Index sorted - stringDVType = DocValuesType.SORTED; - } else { - if (random().nextBoolean()) { - // Index non-sorted - stringDVType = DocValuesType.BINARY; - } else { - // sorted anyway - stringDVType = DocValuesType.SORTED; - } - } - - FieldType ft1 = new FieldType(); - ft1.setStored(true); - FieldType ft2 = new FieldType(); - ft2.setIndexed(true); - for(int i=0; i dirs = new ArrayList(); + /** Tests reverse sorting on type string */ + public void testStringReverse() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "bar", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "foo", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.STRING, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(2, td.totalHits); + // 'foo' comes after 'bar' in reverse order + assertEquals("foo", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value")); + + ir.close(); + dir.close(); + } - @Override - public void tearDown() throws Exception { - full.reader.close(); - searchX.reader.close(); - searchY.reader.close(); - for(Directory dir : dirs) { - dir.close(); + /** Tests sorting on type string_val */ + public void testStringVal() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "foo", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "bar", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.STRING_VAL)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(2, td.totalHits); + // 'bar' comes before 'foo' + assertEquals("bar", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("foo", searcher.doc(td.scoreDocs[1].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type string_val with a missing value */ + public void testStringValMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "foo", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "bar", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.STRING_VAL)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null comes first + assertNull(searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("foo", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests reverse sorting on type string_val */ + public void testStringValReverse() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "bar", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "foo", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.STRING_VAL, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(2, td.totalHits); + // 'foo' comes after 'bar' in reverse order + assertEquals("foo", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on internal docid order */ + public void testFieldDoc() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "foo", Field.Store.NO)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "bar", Field.Store.NO)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(SortField.FIELD_DOC); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(2, td.totalHits); + // docid 0, then docid 1 + assertEquals(0, td.scoreDocs[0].doc); + assertEquals(1, td.scoreDocs[1].doc); + + ir.close(); + dir.close(); + } + + /** Tests sorting on reverse internal docid order */ + public void testFieldDocReverse() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "foo", Field.Store.NO)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "bar", Field.Store.NO)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField(null, SortField.Type.DOC, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(2, td.totalHits); + // docid 1, then docid 0 + assertEquals(1, td.scoreDocs[0].doc); + assertEquals(0, td.scoreDocs[1].doc); + + ir.close(); + dir.close(); + } + + /** Tests default sort (by score) */ + public void testFieldScore() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newTextField("value", "foo bar bar bar bar", Field.Store.NO)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newTextField("value", "foo foo foo foo foo", Field.Store.NO)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(); + + TopDocs actual = searcher.search(new TermQuery(new Term("value", "foo")), 10, sort); + assertEquals(2, actual.totalHits); + + TopDocs expected = searcher.search(new TermQuery(new Term("value", "foo")), 10); + // the two topdocs should be the same + assertEquals(expected.totalHits, actual.totalHits); + for (int i = 0; i < actual.scoreDocs.length; i++) { + assertEquals(actual.scoreDocs[i].doc, expected.scoreDocs[i].doc); } - super.tearDown(); - } - private SortField.Type getDVStringSortType() { - return getDVStringSortType(true); + ir.close(); + dir.close(); } + + /** Tests default sort (by score) in reverse */ + public void testFieldScoreReverse() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newTextField("value", "foo bar bar bar bar", Field.Store.NO)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newTextField("value", "foo foo foo foo foo", Field.Store.NO)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField(null, SortField.Type.SCORE, true)); - private SortField.Type getDVStringSortType(boolean allowSorted) { - if (dvStringSorted && allowSorted) { - // If you index as sorted source you can still sort by - // value instead: - return random().nextBoolean() ? SortField.Type.STRING : SortField.Type.STRING_VAL; - } else { - return SortField.Type.STRING_VAL; + TopDocs actual = searcher.search(new TermQuery(new Term("value", "foo")), 10, sort); + assertEquals(2, actual.totalHits); + + TopDocs expected = searcher.search(new TermQuery(new Term("value", "foo")), 10); + // the two topdocs should be the reverse of each other + assertEquals(expected.totalHits, actual.totalHits); + assertEquals(actual.scoreDocs[0].doc, expected.scoreDocs[1].doc); + assertEquals(actual.scoreDocs[1].doc, expected.scoreDocs[0].doc); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type byte */ + public void testByte() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "23", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.BYTE)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // numeric order + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("23", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type byte with a missing value */ + public void testByteMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.BYTE)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null value is treated as a 0 + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type byte, specifying the missing value should be treated as Byte.MAX_VALUE */ + public void testByteMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + SortField sortField = new SortField("value", SortField.Type.BYTE); + sortField.setMissingValue(Byte.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null value is treated Byte.MAX_VALUE + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type byte in reverse */ + public void testByteReverse() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "23", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.BYTE, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // reverse numeric order + assertEquals("23", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type short */ + public void testShort() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "300", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.SHORT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // numeric order + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("300", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type short with a missing value */ + public void testShortMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.SHORT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as a 0 + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type short, specifying the missing value should be treated as Short.MAX_VALUE */ + public void testShortMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + SortField sortField = new SortField("value", SortField.Type.SHORT); + sortField.setMissingValue(Short.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as Short.MAX_VALUE + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type short in reverse */ + public void testShortReverse() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "300", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.SHORT, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // reverse numeric order + assertEquals("300", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type int */ + public void testInt() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "300000", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.INT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // numeric order + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("300000", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type int with a missing value */ + public void testIntMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.INT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as a 0 + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type int, specifying the missing value should be treated as Integer.MAX_VALUE */ + public void testIntMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + SortField sortField = new SortField("value", SortField.Type.INT); + sortField.setMissingValue(Integer.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as a Integer.MAX_VALUE + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type int in reverse */ + public void testIntReverse() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "300000", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.INT, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // reverse numeric order + assertEquals("300000", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type long */ + public void testLong() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "3000000000", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.LONG)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // numeric order + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("3000000000", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type long with a missing value */ + public void testLongMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.LONG)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as 0 + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type long, specifying the missing value should be treated as Long.MAX_VALUE */ + public void testLongMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + SortField sortField = new SortField("value", SortField.Type.LONG); + sortField.setMissingValue(Long.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as Long.MAX_VALUE + assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type long in reverse */ + public void testLongReverse() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "3000000000", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.LONG, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // reverse numeric order + assertEquals("3000000000", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type float */ + public void testFloat() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "30.1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1.3", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.FLOAT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // numeric order + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("30.1", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type float with a missing value */ + public void testFloatMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1.3", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.FLOAT)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as 0 + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type float, specifying the missing value should be treated as Float.MAX_VALUE */ + public void testFloatMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1.3", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + SortField sortField = new SortField("value", SortField.Type.FLOAT); + sortField.setMissingValue(Float.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // null is treated as Float.MAX_VALUE + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type float in reverse */ + public void testFloatReverse() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "30.1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1.3", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.FLOAT, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(3, td.totalHits); + // reverse numeric order + assertEquals("30.1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("-1.3", searcher.doc(td.scoreDocs[2].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type double */ + public void testDouble() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "30.1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1.3", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.DOUBLE)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(4, td.totalHits); + // numeric order + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[2].doc).get("value")); + assertEquals("30.1", searcher.doc(td.scoreDocs[3].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type double with a missing value */ + public void testDoubleMissing() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1.3", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.DOUBLE)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(4, td.totalHits); + // null treated as a 0 + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[2].doc).get("value")); + assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[3].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type double, specifying the missing value should be treated as Double.MAX_VALUE */ + public void testDoubleMissingLast() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1.3", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + SortField sortField = new SortField("value", SortField.Type.DOUBLE); + sortField.setMissingValue(Double.MAX_VALUE); + Sort sort = new Sort(sortField); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(4, td.totalHits); + // null treated as Double.MAX_VALUE + assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[2].doc).get("value")); + assertNull(searcher.doc(td.scoreDocs[3].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting on type double in reverse */ + public void testDoubleReverse() throws IOException { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "30.1", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "-1.3", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.DOUBLE, true)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(4, td.totalHits); + // numeric order + assertEquals("30.1", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[1].doc).get("value")); + assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[2].doc).get("value")); + assertEquals("-1.3", searcher.doc(td.scoreDocs[3].doc).get("value")); + + ir.close(); + dir.close(); + } + + public void testEmptyStringVsNullStringSort() throws Exception { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random()))); + Document doc = new Document(); + doc.add(newStringField("f", "", Field.Store.NO)); + doc.add(newStringField("t", "1", Field.Store.NO)); + w.addDocument(doc); + w.commit(); + doc = new Document(); + doc.add(newStringField("t", "1", Field.Store.NO)); + w.addDocument(doc); + + IndexReader r = DirectoryReader.open(w, true); + w.close(); + IndexSearcher s = newSearcher(r); + TopDocs hits = s.search(new TermQuery(new Term("t", "1")), null, 10, new Sort(new SortField("f", SortField.Type.STRING))); + assertEquals(2, hits.totalHits); + // null sorts first + assertEquals(1, hits.scoreDocs[0].doc); + assertEquals(0, hits.scoreDocs[1].doc); + r.close(); + dir.close(); + } + + /** test that we don't throw exception on multi-valued field (LUCENE-2142) */ + public void testMultiValuedField() throws IOException { + Directory indexStore = newDirectory(); + IndexWriter writer = new IndexWriter(indexStore, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random()))); + for(int i=0; i<5; i++) { + Document doc = new Document(); + doc.add(new StringField("string", "a"+i, Field.Store.NO)); + doc.add(new StringField("string", "b"+i, Field.Store.NO)); + writer.addDocument(doc); } - } - - /** - * Test String sorting: small queue to many matches, multi field sort, reverse sort - */ - public void testStringSort() throws Exception { - // Normal string field, var length - sort.setSort( + writer.forceMerge(1); // enforce one segment to have a higher unique term count in all cases + writer.close(); + Sort sort = new Sort( new SortField("string", SortField.Type.STRING), - new SortField("string2", SortField.Type.STRING, true), SortField.FIELD_DOC); - verifyStringSort(sort); - - // Normal string field, fixed length - sort.setSort( - new SortField("string_fixed", SortField.Type.STRING), - new SortField("string2_fixed", SortField.Type.STRING, true), - SortField.FIELD_DOC); - verifyStringSort(sort); - - // Doc values field, var length - sort.setSort( - new SortField("string_dv", getDVStringSortType()), - new SortField("string2_dv", getDVStringSortType(), true), - SortField.FIELD_DOC); - verifyStringSort(sort); - - // Doc values field, fixed length - sort.setSort( - new SortField("string_fixed_dv", getDVStringSortType()), - new SortField("string2_fixed_dv", getDVStringSortType(), true), - SortField.FIELD_DOC); - verifyStringSort(sort); + // this should not throw AIOOBE or RuntimeEx + IndexReader reader = DirectoryReader.open(indexStore); + IndexSearcher searcher = new IndexSearcher(reader); + searcher.search(new MatchAllDocsQuery(), null, 500, sort); + reader.close(); + indexStore.close(); } - - private void verifyStringSort(Sort sort) throws Exception { - if (VERBOSE) { - System.out.println("verifySort sort=" + sort); - } - final IndexSearcher searcher = getFullStrings(); - final ScoreDoc[] result = searcher.search(new MatchAllDocsQuery(), null, _TestUtil.nextInt(random(), 500, searcher.getIndexReader().maxDoc()), sort).scoreDocs; - StringBuilder buff = new StringBuilder(); - int n = result.length; - String last = null; - String lastSub = null; - int lastDocId = 0; - boolean fail = false; - final String fieldSuffix = (sort.getSort()[0].getField().indexOf("_fixed") != -1) ? "_fixed" : ""; - for(int x = 0; x < n; ++x) { - StoredDocument doc2 = searcher.doc(result[x].doc); - StorableField[] v = doc2.getFields("tracer" + fieldSuffix); - StorableField[] v2 = doc2.getFields("tracer2" + fieldSuffix); - for(int j = 0; j < v.length; ++j) { - buff.append(v[j].stringValue() + "(" + v2[j].stringValue() + ")(" + result[x].doc+")\n"); - if (last != null) { - int cmp = v[j].stringValue().compareTo(last); - if (!(cmp >= 0)) { // ensure first field is in order - fail = true; - System.out.println("fail:" + v[j] + " < " + last); - buff.append(" WRONG tracer\n"); - } - if (cmp == 0) { // ensure second field is in reverse order - cmp = v2[j].stringValue().compareTo(lastSub); - if (cmp > 0) { - fail = true; - System.out.println("rev field fail:" + v2[j] + " > " + lastSub); - buff.append(" WRONG tracer2\n"); - } else if(cmp == 0) { // ensure docid is in order - if (result[x].doc < lastDocId) { - fail = true; - System.out.println("doc fail:" + result[x].doc + " > " + lastDocId); - buff.append(" WRONG docID\n"); - } - } - } + + public void testMaxScore() throws Exception { + Directory d = newDirectory(); + // Not RIW because we need exactly 2 segs: + IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); + int id = 0; + for(int seg=0;seg<2;seg++) { + for(int docIDX=0;docIDX<10;docIDX++) { + Document doc = new Document(); + doc.add(newStringField("id", ""+docIDX, Field.Store.YES)); + StringBuilder sb = new StringBuilder(); + for(int i=0;i { - FieldCache.Ints docValues; - int[] slotValues; - int bottomValue; + sort.setSort(SortField.FIELD_DOC); + td = empty.search(query, null, 10, sort, true, true); + assertEquals(0, td.totalHits); - MyFieldComparator(int numHits) { - slotValues = new int[numHits]; + sort.setSort(new SortField("int", SortField.Type.INT), SortField.FIELD_DOC); + td = empty.search(query, null, 10, sort, true, true); + assertEquals(0, td.totalHits); + + sort.setSort(new SortField("string", SortField.Type.STRING, true), SortField.FIELD_DOC); + td = empty.search(query, null, 10, sort, true, true); + assertEquals(0, td.totalHits); + + sort.setSort(new SortField("string_val", SortField.Type.STRING_VAL, true), SortField.FIELD_DOC); + td = empty.search(query, null, 10, sort, true, true); + assertEquals(0, td.totalHits); + + sort.setSort(new SortField("float", SortField.Type.FLOAT), new SortField("string", SortField.Type.STRING)); + td = empty.search(query, null, 10, sort, true, true); + assertEquals(0, td.totalHits); + } + + /** + * test sorts for a custom int parser that uses a simple char encoding + */ + public void testCustomIntParser() throws Exception { + List letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); + Collections.shuffle(letters, random()); + + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + for (String letter : letters) { + Document doc = new Document(); + doc.add(newStringField("parser", letter, Field.Store.YES)); + iw.addDocument(doc); } - - @Override - public void copy(int slot, int doc) { - slotValues[slot] = docValues.get(doc); - } - - @Override - public int compare(int slot1, int slot2) { - // values are small enough that overflow won't happen - return slotValues[slot1] - slotValues[slot2]; - } - - @Override - public int compareBottom(int doc) { - return bottomValue - docValues.get(doc); - } - - @Override - public void setBottom(int bottom) { - bottomValue = slotValues[bottom]; - } - - private static final FieldCache.IntParser testIntParser = new FieldCache.IntParser() { + + IndexReader ir = iw.getReader(); + iw.close(); + + IndexSearcher searcher = newSearcher(ir); + Sort sort = new Sort(new SortField("parser", new FieldCache.IntParser() { @Override - public final int parseInt(final BytesRef term) { + public int parseInt(BytesRef term) { return (term.bytes[term.offset]-'A') * 123456; } @@ -514,301 +1207,313 @@ public class TestSort extends LuceneTestCase { public TermsEnum termsEnum(Terms terms) throws IOException { return terms.iterator(null); } - }; - - @Override - public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - docValues = FieldCache.DEFAULT.getInts(context.reader(), "parser", testIntParser, false); - return this; - } - - @Override - public Integer value(int slot) { - return Integer.valueOf(slotValues[slot]); - } - - @Override - public int compareDocToValue(int doc, Integer valueObj) { - final int value = valueObj.intValue(); - final int docValue = docValues.get(doc); - - // values are small enough that overflow won't happen - return docValue - value; - } - } - - static class MyFieldComparatorSource extends FieldComparatorSource { - @Override - public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) { - return new MyFieldComparator(numHits); - } - } - - // Test sorting w/ custom FieldComparator - public void testNewCustomFieldParserSort() throws Exception { - sort.setSort(new SortField("parser", new MyFieldComparatorSource())); - assertMatches(full, queryA, sort, "JIHGFEDCBA"); - } - - // test sorting when the sort field is empty (undefined) for some of the documents - public void testEmptyFieldSort() throws Exception { - - // NOTE: do not test DocValues fields here, since you - // can't sort when some documents don't have the field - sort.setSort(new SortField("string", SortField.Type.STRING)); - assertMatches(full, queryF, sort, "ZJI"); - - sort.setSort(new SortField("string", SortField.Type.STRING, true)); - assertMatches(full, queryF, sort, "IJZ"); + }), SortField.FIELD_DOC ); - sort.setSort(new SortField("int", SortField.Type.INT)); - assertMatches(full, queryF, sort, "IZJ"); + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - sort.setSort(new SortField("int", SortField.Type.INT, true)); - assertMatches(full, queryF, sort, "JZI"); - - sort.setSort(new SortField("float", SortField.Type.FLOAT)); - assertMatches(full, queryF, sort, "ZJI"); - - // using a nonexisting field as first sort key shouldn't make a difference: - sort.setSort(new SortField("nosuchfield", SortField.Type.STRING), - new SortField("float", SortField.Type.FLOAT)); - assertMatches(full, queryF, sort, "ZJI"); - - sort.setSort(new SortField("float", SortField.Type.FLOAT, true)); - assertMatches(full, queryF, sort, "IJZ"); - - // When a field is null for both documents, the next SortField should be used. - sort.setSort(new SortField("int", SortField.Type.INT), - new SortField("string", SortField.Type.STRING), - new SortField("float", SortField.Type.FLOAT)); - assertMatches(full, queryG, sort, "ZWXY"); - - // Reverse the last criterium to make sure the test didn't pass by chance - sort.setSort(new SortField("int", SortField.Type.INT), - new SortField("string", SortField.Type.STRING), - new SortField("float", SortField.Type.FLOAT, true)); - assertMatches(full, queryG, sort, "ZYXW"); - - // Do the same for a ParallelMultiSearcher - ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random(), 2, 8), new NamedThreadFactory("testEmptyFieldSort")); - IndexSearcher parallelSearcher = new IndexSearcher(full.getIndexReader(), exec); - - sort.setSort(new SortField("int", SortField.Type.INT), - new SortField("string", SortField.Type.STRING), - new SortField("float", SortField.Type.FLOAT)); - assertMatches(parallelSearcher, queryG, sort, "ZWXY"); - - sort.setSort(new SortField("int", SortField.Type.INT), - new SortField("string", SortField.Type.STRING), - new SortField("float", SortField.Type.FLOAT, true)); - assertMatches(parallelSearcher, queryG, sort, "ZYXW"); - exec.shutdown(); - exec.awaitTermination(1000, TimeUnit.MILLISECONDS); - } - - // test sorts using a series of fields - public void testSortCombos() throws Exception { - sort.setSort(new SortField("int", SortField.Type.INT), new SortField("float", SortField.Type.FLOAT)); - assertMatches(full, queryX, sort, "IGEAC"); - - sort.setSort(new SortField("int", SortField.Type.INT, true), new SortField(null, SortField.Type.DOC, true)); - assertMatches(full, queryX, sort, "CEAGI"); - - sort.setSort(new SortField("float", SortField.Type.FLOAT), new SortField("string", SortField.Type.STRING)); - assertMatches(full, queryX, sort, "GICEA"); - - sort.setSort(new SortField("int_dv", SortField.Type.INT), - new SortField("float_dv", SortField.Type.FLOAT)); - assertMatches(full, queryX, sort, "IGEAC"); - - sort.setSort(new SortField("int_dv", SortField.Type.INT, true), - new SortField(null, SortField.Type.DOC, true)); - assertMatches(full, queryX, sort, "CEAGI"); - - sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), - new SortField("string_dv", getDVStringSortType())); - assertMatches(full, queryX, sort, "GICEA"); - } - - // test a variety of sorts using a parallel multisearcher - public void testParallelMultiSort() throws Exception { - ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random(), 2, 8), new NamedThreadFactory("testParallelMultiSort")); - IndexSearcher searcher = new IndexSearcher( - new MultiReader(searchX.getIndexReader(), - searchY.getIndexReader()), exec); - try { - runMultiSorts(searcher, false); - } finally { - exec.shutdown(); - exec.awaitTermination(1000, TimeUnit.MILLISECONDS); + // results should be in alphabetical order + assertEquals(10, td.totalHits); + Collections.sort(letters); + for (int i = 0; i < letters.size(); i++) { + assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); } - } - public void testTopDocsScores() throws Exception { - - // There was previously a bug in FieldSortedHitQueue.maxscore when only a single - // doc was added. That is what the following tests for. - Sort sort = new Sort(); - int nDocs=10; - - // try to pick a query that will result in an unnormalized - // score greater than 1 to test for correct normalization - final TopDocs docs1 = full.search(queryE,null,nDocs,sort,true,true); - - // a filter that only allows through the first hit - Filter filt = new Filter() { - @Override - public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) { - assertNull("acceptDocs should be null, as we have no deletions", acceptDocs); - BitSet bs = new BitSet(context.reader().maxDoc()); - bs.set(0, context.reader().maxDoc()); - bs.set(docs1.scoreDocs[0].doc); - return new DocIdBitSet(bs); - } - }; - - TopDocs docs2 = full.search(queryE, filt, nDocs, sort,true,true); - - assertEquals(docs1.scoreDocs[0].score, docs2.scoreDocs[0].score, 1e-6); + ir.close(); + dir.close(); } - // runs a variety of sorts useful for multisearchers - private void runMultiSorts(IndexSearcher multi, boolean isFull) throws Exception { - sort.setSort(SortField.FIELD_DOC); - String expected = isFull ? "ABCDEFGHIJ" : "ACEGIBDFHJ"; - assertMatches(multi, queryA, sort, expected); + /** + * test sorts for a custom byte parser that uses a simple char encoding + */ + public void testCustomByteParser() throws Exception { + List letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); + Collections.shuffle(letters, random()); - sort.setSort(new SortField("int", SortField.Type.INT)); - expected = isFull ? "IDHFGJABEC" : "IDHFGJAEBC"; - assertMatches(multi, queryA, sort, expected); - - sort.setSort(new SortField("int", SortField.Type.INT), SortField.FIELD_DOC); - expected = isFull ? "IDHFGJABEC" : "IDHFGJAEBC"; - assertMatches(multi, queryA, sort, expected); - - sort.setSort(new SortField("int", SortField.Type.INT)); - expected = isFull ? "IDHFGJABEC" : "IDHFGJAEBC"; - assertMatches(multi, queryA, sort, expected); - - sort.setSort(new SortField("float", SortField.Type.FLOAT), SortField.FIELD_DOC); - assertMatches(multi, queryA, sort, "GDHJCIEFAB"); - - sort.setSort(new SortField("float", SortField.Type.FLOAT)); - assertMatches(multi, queryA, sort, "GDHJCIEFAB"); - - sort.setSort(new SortField("string", SortField.Type.STRING)); - assertMatches(multi, queryA, sort, "DJAIHGFEBC"); - - sort.setSort(new SortField("int", SortField.Type.INT, true)); - expected = isFull ? "CABEJGFHDI" : "CAEBJGFHDI"; - assertMatches(multi, queryA, sort, expected); - - sort.setSort(new SortField("float", SortField.Type.FLOAT, true)); - assertMatches(multi, queryA, sort, "BAFECIJHDG"); - - sort.setSort(new SortField("string", SortField.Type.STRING, true)); - assertMatches(multi, queryA, sort, "CBEFGHIAJD"); - - sort.setSort(new SortField("int", SortField.Type.INT),new SortField("float", SortField.Type.FLOAT)); - assertMatches(multi, queryA, sort, "IDHFGJEABC"); - - sort.setSort(new SortField("float", SortField.Type.FLOAT),new SortField("string", SortField.Type.STRING)); - assertMatches(multi, queryA, sort, "GDHJICEFAB"); - - sort.setSort(new SortField("int", SortField.Type.INT)); - assertMatches(multi, queryF, sort, "IZJ"); - - sort.setSort(new SortField("int", SortField.Type.INT, true)); - assertMatches(multi, queryF, sort, "JZI"); - - sort.setSort(new SortField("float", SortField.Type.FLOAT)); - assertMatches(multi, queryF, sort, "ZJI"); - - sort.setSort(new SortField("string", SortField.Type.STRING)); - assertMatches(multi, queryF, sort, "ZJI"); - - sort.setSort(new SortField("string", SortField.Type.STRING, true)); - assertMatches(multi, queryF, sort, "IJZ"); - - sort.setSort(new SortField("int_dv", SortField.Type.INT)); - expected = isFull ? "IDHFGJABEC" : "IDHFGJAEBC"; - assertMatches(multi, queryA, sort, expected); - - sort.setSort(new SortField("int_dv", SortField.Type.INT), SortField.FIELD_DOC); - expected = isFull ? "IDHFGJABEC" : "IDHFGJAEBC"; - assertMatches(multi, queryA, sort, expected); - - sort.setSort(new SortField("int_dv", SortField.Type.INT)); - expected = isFull ? "IDHFGJABEC" : "IDHFGJAEBC"; - assertMatches(multi, queryA, sort, expected); - - sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), SortField.FIELD_DOC); - assertMatches(multi, queryA, sort, "GDHJCIEFAB"); - - sort.setSort(new SortField("float_dv", SortField.Type.FLOAT)); - assertMatches(multi, queryA, sort, "GDHJCIEFAB"); - - sort.setSort(new SortField("int_dv", SortField.Type.INT, true)); - expected = isFull ? "CABEJGFHDI" : "CAEBJGFHDI"; - assertMatches(multi, queryA, sort, expected); - - sort.setSort(new SortField("int_dv", SortField.Type.INT), new SortField("float_dv", SortField.Type.FLOAT)); - assertMatches(multi, queryA, sort, "IDHFGJEABC"); - - sort.setSort(new SortField("int_dv", SortField.Type.INT)); - assertMatches(multi, queryF, sort, "IZJ"); - - sort.setSort(new SortField("int_dv", SortField.Type.INT, true)); - assertMatches(multi, queryF, sort, "JZI"); - - sort.setSort(new SortField("string_dv", getDVStringSortType())); - assertMatches(multi, queryA, sort, "DJAIHGFEBC"); - - sort.setSort(new SortField("string_dv", getDVStringSortType(), true)); - assertMatches(multi, queryA, sort, "CBEFGHIAJD"); - - sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), new SortField("string_dv", getDVStringSortType())); - assertMatches(multi, queryA, sort, "GDHJICEFAB"); - - sort.setSort(new SortField("string_dv", getDVStringSortType())); - assertMatches(multi, queryF, sort, "ZJI"); - - sort.setSort(new SortField("string_dv", getDVStringSortType(), true)); - assertMatches(multi, queryF, sort, "IJZ"); - - // up to this point, all of the searches should have "sane" - // FieldCache behavior, and should have reused hte cache in several cases - assertSaneFieldCaches(getTestName() + " various"); - // next we'll check Locale based (String[]) for 'string', so purge first - FieldCache.DEFAULT.purgeAllCaches(); - } - - private void assertMatches(IndexSearcher searcher, Query query, Sort sort, String expectedResult) throws IOException { - assertMatches( null, searcher, query, sort, expectedResult); - } - - - // make sure the documents returned by the search match the expected list - private void assertMatches(String msg, IndexSearcher searcher, Query query, Sort sort, - String expectedResult) throws IOException { - if (VERBOSE) { - System.out.println("assertMatches searcher=" + searcher + " sort=" + sort); + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + for (String letter : letters) { + Document doc = new Document(); + doc.add(newStringField("parser", letter, Field.Store.YES)); + iw.addDocument(doc); } - - //ScoreDoc[] result = searcher.search (query, null, 1000, sort).scoreDocs; - TopDocs hits = searcher.search(query, null, Math.max(1, expectedResult.length()), sort, true, true); - ScoreDoc[] result = hits.scoreDocs; - assertEquals(expectedResult.length(),hits.totalHits); - StringBuilder buff = new StringBuilder(10); - int n = result.length; - for(int i=0; i letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); + Collections.shuffle(letters, random()); + + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + for (String letter : letters) { + Document doc = new Document(); + doc.add(newStringField("parser", letter, Field.Store.YES)); + iw.addDocument(doc); + } + + IndexReader ir = iw.getReader(); + iw.close(); + + IndexSearcher searcher = newSearcher(ir); + Sort sort = new Sort(new SortField("parser", new FieldCache.ShortParser() { + @Override + public short parseShort(BytesRef term) { + return (short) (term.bytes[term.offset]-'A'); + } + + @Override + public TermsEnum termsEnum(Terms terms) throws IOException { + return terms.iterator(null); + } + }), SortField.FIELD_DOC ); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + + // results should be in alphabetical order + assertEquals(10, td.totalHits); + Collections.sort(letters); + for (int i = 0; i < letters.size(); i++) { + assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); + } + + ir.close(); + dir.close(); + } + + /** + * test sorts for a custom long parser that uses a simple char encoding + */ + public void testCustomLongParser() throws Exception { + List letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); + Collections.shuffle(letters, random()); + + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + for (String letter : letters) { + Document doc = new Document(); + doc.add(newStringField("parser", letter, Field.Store.YES)); + iw.addDocument(doc); + } + + IndexReader ir = iw.getReader(); + iw.close(); + + IndexSearcher searcher = newSearcher(ir); + Sort sort = new Sort(new SortField("parser", new FieldCache.LongParser() { + @Override + public long parseLong(BytesRef term) { + return (term.bytes[term.offset]-'A') * 1234567890L; + } + + @Override + public TermsEnum termsEnum(Terms terms) throws IOException { + return terms.iterator(null); + } + }), SortField.FIELD_DOC ); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + + // results should be in alphabetical order + assertEquals(10, td.totalHits); + Collections.sort(letters); + for (int i = 0; i < letters.size(); i++) { + assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); + } + + ir.close(); + dir.close(); + } + + /** + * test sorts for a custom float parser that uses a simple char encoding + */ + public void testCustomFloatParser() throws Exception { + List letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); + Collections.shuffle(letters, random()); + + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + for (String letter : letters) { + Document doc = new Document(); + doc.add(newStringField("parser", letter, Field.Store.YES)); + iw.addDocument(doc); + } + + IndexReader ir = iw.getReader(); + iw.close(); + + IndexSearcher searcher = newSearcher(ir); + Sort sort = new Sort(new SortField("parser", new FieldCache.FloatParser() { + @Override + public float parseFloat(BytesRef term) { + return (float) Math.sqrt(term.bytes[term.offset]); + } + + @Override + public TermsEnum termsEnum(Terms terms) throws IOException { + return terms.iterator(null); + } + }), SortField.FIELD_DOC ); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + + // results should be in alphabetical order + assertEquals(10, td.totalHits); + Collections.sort(letters); + for (int i = 0; i < letters.size(); i++) { + assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); + } + + ir.close(); + dir.close(); + } + + /** + * test sorts for a custom double parser that uses a simple char encoding + */ + public void testCustomDoubleParser() throws Exception { + List letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); + Collections.shuffle(letters, random()); + + Directory dir = newDirectory(); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir); + for (String letter : letters) { + Document doc = new Document(); + doc.add(newStringField("parser", letter, Field.Store.YES)); + iw.addDocument(doc); + } + + IndexReader ir = iw.getReader(); + iw.close(); + + IndexSearcher searcher = newSearcher(ir); + Sort sort = new Sort(new SortField("parser", new FieldCache.DoubleParser() { + @Override + public double parseDouble(BytesRef term) { + return Math.pow(term.bytes[term.offset], (term.bytes[term.offset]-'A')); + } + + @Override + public TermsEnum termsEnum(Terms terms) throws IOException { + return terms.iterator(null); + } + }), SortField.FIELD_DOC ); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + + // results should be in alphabetical order + assertEquals(10, td.totalHits); + Collections.sort(letters); + for (int i = 0; i < letters.size(); i++) { + assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); + } + + ir.close(); + dir.close(); + } + + /** Tests sorting a single document */ + public void testSortOneDocument() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "foo", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.STRING)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(1, td.totalHits); + assertEquals("foo", searcher.doc(td.scoreDocs[0].doc).get("value")); + + ir.close(); + dir.close(); + } + + /** Tests sorting a single document with scores */ + public void testSortOneDocumentWithScores() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("value", "foo", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + Sort sort = new Sort(new SortField("value", SortField.Type.STRING)); + + TopDocs expected = searcher.search(new TermQuery(new Term("value", "foo")), 10); + assertEquals(1, expected.totalHits); + TopDocs actual = searcher.search(new TermQuery(new Term("value", "foo")), null, 10, sort, true, true); + + assertEquals(expected.totalHits, actual.totalHits); + assertEquals(expected.scoreDocs[0].score, actual.scoreDocs[0].score, 0F); + + ir.close(); + dir.close(); + } + + /** Tests sorting with two fields */ + public void testSortTwoFields() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + Document doc = new Document(); + doc.add(newStringField("tievalue", "tied", Field.Store.NO)); + doc.add(newStringField("value", "foo", Field.Store.YES)); + writer.addDocument(doc); + doc = new Document(); + doc.add(newStringField("tievalue", "tied", Field.Store.NO)); + doc.add(newStringField("value", "bar", Field.Store.YES)); + writer.addDocument(doc); + IndexReader ir = writer.getReader(); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(ir); + // tievalue, then value + Sort sort = new Sort(new SortField("tievalue", SortField.Type.STRING), + new SortField("value", SortField.Type.STRING)); + + TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); + assertEquals(2, td.totalHits); + // 'bar' comes before 'foo' + assertEquals("bar", searcher.doc(td.scoreDocs[0].doc).get("value")); + assertEquals("foo", searcher.doc(td.scoreDocs[1].doc).get("value")); + + ir.close(); + dir.close(); } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSort2.java b/lucene/core/src/test/org/apache/lucene/search/TestSort2.java deleted file mode 100644 index 52c22bc8cfd..00000000000 --- a/lucene/core/src/test/org/apache/lucene/search/TestSort2.java +++ /dev/null @@ -1,1519 +0,0 @@ -package org.apache.lucene.search; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.MultiReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LuceneTestCase; - -/* - * Very simple tests of sorting. - * - * THE RULES: - * 1. keywords like 'abstract' and 'static' should not appear in this file. - * 2. each test method should be self-contained and understandable. - * 3. no test methods should share code with other test methods. - * 4. no testing of things unrelated to sorting. - * 5. no tracers. - * 6. keyword 'class' should appear only once in this file, here ---- - * | - * ----------------------------------------------------------- - * | - * \./ - */ -public class TestSort2 extends LuceneTestCase { - - /** Tests sorting on type string */ - public void testString() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "foo", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "bar", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.STRING)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(2, td.totalHits); - // 'bar' comes before 'foo' - assertEquals("bar", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("foo", searcher.doc(td.scoreDocs[1].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type string with a missing value */ - public void testStringMissing() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "foo", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "bar", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.STRING)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // null comes first - assertNull(searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("foo", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests reverse sorting on type string */ - public void testStringReverse() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "bar", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "foo", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.STRING, true)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(2, td.totalHits); - // 'foo' comes after 'bar' in reverse order - assertEquals("foo", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type string_val */ - public void testStringVal() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "foo", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "bar", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.STRING_VAL)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(2, td.totalHits); - // 'bar' comes before 'foo' - assertEquals("bar", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("foo", searcher.doc(td.scoreDocs[1].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type string_val with a missing value */ - public void testStringValMissing() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "foo", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "bar", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.STRING_VAL)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // null comes first - assertNull(searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("foo", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests reverse sorting on type string_val */ - public void testStringValReverse() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "bar", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "foo", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.STRING_VAL, true)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(2, td.totalHits); - // 'foo' comes after 'bar' in reverse order - assertEquals("foo", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("bar", searcher.doc(td.scoreDocs[1].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on internal docid order */ - public void testFieldDoc() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "foo", Field.Store.NO)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "bar", Field.Store.NO)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(SortField.FIELD_DOC); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(2, td.totalHits); - // docid 0, then docid 1 - assertEquals(0, td.scoreDocs[0].doc); - assertEquals(1, td.scoreDocs[1].doc); - - ir.close(); - dir.close(); - } - - /** Tests sorting on reverse internal docid order */ - public void testFieldDocReverse() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "foo", Field.Store.NO)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "bar", Field.Store.NO)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField(null, SortField.Type.DOC, true)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(2, td.totalHits); - // docid 1, then docid 0 - assertEquals(1, td.scoreDocs[0].doc); - assertEquals(0, td.scoreDocs[1].doc); - - ir.close(); - dir.close(); - } - - /** Tests default sort (by score) */ - public void testFieldScore() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newTextField("value", "foo bar bar bar bar", Field.Store.NO)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newTextField("value", "foo foo foo foo foo", Field.Store.NO)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(); - - TopDocs actual = searcher.search(new TermQuery(new Term("value", "foo")), 10, sort); - assertEquals(2, actual.totalHits); - - TopDocs expected = searcher.search(new TermQuery(new Term("value", "foo")), 10); - // the two topdocs should be the same - assertEquals(expected.totalHits, actual.totalHits); - for (int i = 0; i < actual.scoreDocs.length; i++) { - assertEquals(actual.scoreDocs[i].doc, expected.scoreDocs[i].doc); - } - - ir.close(); - dir.close(); - } - - /** Tests default sort (by score) in reverse */ - public void testFieldScoreReverse() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newTextField("value", "foo bar bar bar bar", Field.Store.NO)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newTextField("value", "foo foo foo foo foo", Field.Store.NO)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField(null, SortField.Type.SCORE, true)); - - TopDocs actual = searcher.search(new TermQuery(new Term("value", "foo")), 10, sort); - assertEquals(2, actual.totalHits); - - TopDocs expected = searcher.search(new TermQuery(new Term("value", "foo")), 10); - // the two topdocs should be the reverse of each other - assertEquals(expected.totalHits, actual.totalHits); - assertEquals(actual.scoreDocs[0].doc, expected.scoreDocs[1].doc); - assertEquals(actual.scoreDocs[1].doc, expected.scoreDocs[0].doc); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type byte */ - public void testByte() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "23", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.BYTE)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // numeric order - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("23", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type byte with a missing value */ - public void testByteMissing() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.BYTE)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // null value is treated as a 0 - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type byte, specifying the missing value should be treated as Byte.MAX_VALUE */ - public void testByteMissingLast() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - SortField sortField = new SortField("value", SortField.Type.BYTE); - sortField.setMissingValue(Byte.MAX_VALUE); - Sort sort = new Sort(sortField); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // null value is treated Byte.MAX_VALUE - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type byte in reverse */ - public void testByteReverse() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "23", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.BYTE, true)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // reverse numeric order - assertEquals("23", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type short */ - public void testShort() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "300", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.SHORT)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // numeric order - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("300", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type short with a missing value */ - public void testShortMissing() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.SHORT)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // null is treated as a 0 - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type short, specifying the missing value should be treated as Short.MAX_VALUE */ - public void testShortMissingLast() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - SortField sortField = new SortField("value", SortField.Type.SHORT); - sortField.setMissingValue(Short.MAX_VALUE); - Sort sort = new Sort(sortField); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // null is treated as Short.MAX_VALUE - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type short in reverse */ - public void testShortReverse() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "300", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.SHORT, true)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // reverse numeric order - assertEquals("300", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type int */ - public void testInt() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "300000", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.INT)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // numeric order - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("300000", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type int with a missing value */ - public void testIntMissing() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.INT)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // null is treated as a 0 - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type int, specifying the missing value should be treated as Integer.MAX_VALUE */ - public void testIntMissingLast() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - SortField sortField = new SortField("value", SortField.Type.INT); - sortField.setMissingValue(Integer.MAX_VALUE); - Sort sort = new Sort(sortField); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // null is treated as a Integer.MAX_VALUE - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type int in reverse */ - public void testIntReverse() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "300000", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.INT, true)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // reverse numeric order - assertEquals("300000", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type long */ - public void testLong() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "3000000000", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.LONG)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // numeric order - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("3000000000", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type long with a missing value */ - public void testLongMissing() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.LONG)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // null is treated as 0 - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type long, specifying the missing value should be treated as Long.MAX_VALUE */ - public void testLongMissingLast() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - SortField sortField = new SortField("value", SortField.Type.LONG); - sortField.setMissingValue(Long.MAX_VALUE); - Sort sort = new Sort(sortField); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // null is treated as Long.MAX_VALUE - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type long in reverse */ - public void testLongReverse() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "3000000000", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.LONG, true)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // reverse numeric order - assertEquals("3000000000", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type float */ - public void testFloat() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "30.1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1.3", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4.2", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.FLOAT)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // numeric order - assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("30.1", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type float with a missing value */ - public void testFloatMissing() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1.3", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4.2", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.FLOAT)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // null is treated as 0 - assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("4.2", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type float, specifying the missing value should be treated as Float.MAX_VALUE */ - public void testFloatMissingLast() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1.3", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4.2", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - SortField sortField = new SortField("value", SortField.Type.FLOAT); - sortField.setMissingValue(Float.MAX_VALUE); - Sort sort = new Sort(sortField); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // null is treated as Float.MAX_VALUE - assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertNull(searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type float in reverse */ - public void testFloatReverse() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "30.1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1.3", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4.2", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.FLOAT, true)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // reverse numeric order - assertEquals("30.1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("-1.3", searcher.doc(td.scoreDocs[2].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type double */ - public void testDouble() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "30.1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1.3", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.DOUBLE)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(4, td.totalHits); - // numeric order - assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[2].doc).get("value")); - assertEquals("30.1", searcher.doc(td.scoreDocs[3].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type double with a missing value */ - public void testDoubleMissing() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1.3", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.DOUBLE)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(4, td.totalHits); - // null treated as a 0 - assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertNull(searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[2].doc).get("value")); - assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[3].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type double, specifying the missing value should be treated as Double.MAX_VALUE */ - public void testDoubleMissingLast() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1.3", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - SortField sortField = new SortField("value", SortField.Type.DOUBLE); - sortField.setMissingValue(Double.MAX_VALUE); - Sort sort = new Sort(sortField); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(4, td.totalHits); - // null treated as Double.MAX_VALUE - assertEquals("-1.3", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[2].doc).get("value")); - assertNull(searcher.doc(td.scoreDocs[3].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type double in reverse */ - public void testDoubleReverse() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "30.1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "-1.3", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.DOUBLE, true)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(4, td.totalHits); - // numeric order - assertEquals("30.1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4.2333333333333", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("4.2333333333332", searcher.doc(td.scoreDocs[2].doc).get("value")); - assertEquals("-1.3", searcher.doc(td.scoreDocs[3].doc).get("value")); - - ir.close(); - dir.close(); - } - - public void testEmptyStringVsNullStringSort() throws Exception { - Directory dir = newDirectory(); - IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random()))); - Document doc = new Document(); - doc.add(newStringField("f", "", Field.Store.NO)); - doc.add(newStringField("t", "1", Field.Store.NO)); - w.addDocument(doc); - w.commit(); - doc = new Document(); - doc.add(newStringField("t", "1", Field.Store.NO)); - w.addDocument(doc); - - IndexReader r = DirectoryReader.open(w, true); - w.close(); - IndexSearcher s = newSearcher(r); - TopDocs hits = s.search(new TermQuery(new Term("t", "1")), null, 10, new Sort(new SortField("f", SortField.Type.STRING))); - assertEquals(2, hits.totalHits); - // null sorts first - assertEquals(1, hits.scoreDocs[0].doc); - assertEquals(0, hits.scoreDocs[1].doc); - r.close(); - dir.close(); - } - - /** test that we don't throw exception on multi-valued field (LUCENE-2142) */ - public void testMultiValuedField() throws IOException { - Directory indexStore = newDirectory(); - IndexWriter writer = new IndexWriter(indexStore, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random()))); - for(int i=0; i<5; i++) { - Document doc = new Document(); - doc.add(new StringField("string", "a"+i, Field.Store.NO)); - doc.add(new StringField("string", "b"+i, Field.Store.NO)); - writer.addDocument(doc); - } - writer.forceMerge(1); // enforce one segment to have a higher unique term count in all cases - writer.close(); - Sort sort = new Sort( - new SortField("string", SortField.Type.STRING), - SortField.FIELD_DOC); - // this should not throw AIOOBE or RuntimeEx - IndexReader reader = DirectoryReader.open(indexStore); - IndexSearcher searcher = new IndexSearcher(reader); - searcher.search(new MatchAllDocsQuery(), null, 500, sort); - reader.close(); - indexStore.close(); - } - - public void testMaxScore() throws Exception { - Directory d = newDirectory(); - // Not RIW because we need exactly 2 segs: - IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); - int id = 0; - for(int seg=0;seg<2;seg++) { - for(int docIDX=0;docIDX<10;docIDX++) { - Document doc = new Document(); - doc.add(newStringField("id", ""+docIDX, Field.Store.YES)); - StringBuilder sb = new StringBuilder(); - for(int i=0;i letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); - Collections.shuffle(letters, random()); - - Directory dir = newDirectory(); - RandomIndexWriter iw = new RandomIndexWriter(random(), dir); - for (String letter : letters) { - Document doc = new Document(); - doc.add(newStringField("parser", letter, Field.Store.YES)); - iw.addDocument(doc); - } - - IndexReader ir = iw.getReader(); - iw.close(); - - IndexSearcher searcher = newSearcher(ir); - Sort sort = new Sort(new SortField("parser", new FieldCache.IntParser() { - @Override - public int parseInt(BytesRef term) { - return (term.bytes[term.offset]-'A') * 123456; - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - }), SortField.FIELD_DOC ); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - - // results should be in alphabetical order - assertEquals(10, td.totalHits); - Collections.sort(letters); - for (int i = 0; i < letters.size(); i++) { - assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); - } - - ir.close(); - dir.close(); - } - - /** - * test sorts for a custom byte parser that uses a simple char encoding - */ - public void testCustomByteParser() throws Exception { - List letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); - Collections.shuffle(letters, random()); - - Directory dir = newDirectory(); - RandomIndexWriter iw = new RandomIndexWriter(random(), dir); - for (String letter : letters) { - Document doc = new Document(); - doc.add(newStringField("parser", letter, Field.Store.YES)); - iw.addDocument(doc); - } - - IndexReader ir = iw.getReader(); - iw.close(); - - IndexSearcher searcher = newSearcher(ir); - Sort sort = new Sort(new SortField("parser", new FieldCache.ByteParser() { - @Override - public byte parseByte(BytesRef term) { - return (byte) (term.bytes[term.offset]-'A'); - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - }), SortField.FIELD_DOC ); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - - // results should be in alphabetical order - assertEquals(10, td.totalHits); - Collections.sort(letters); - for (int i = 0; i < letters.size(); i++) { - assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); - } - - ir.close(); - dir.close(); - } - - /** - * test sorts for a custom short parser that uses a simple char encoding - */ - public void testCustomShortParser() throws Exception { - List letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); - Collections.shuffle(letters, random()); - - Directory dir = newDirectory(); - RandomIndexWriter iw = new RandomIndexWriter(random(), dir); - for (String letter : letters) { - Document doc = new Document(); - doc.add(newStringField("parser", letter, Field.Store.YES)); - iw.addDocument(doc); - } - - IndexReader ir = iw.getReader(); - iw.close(); - - IndexSearcher searcher = newSearcher(ir); - Sort sort = new Sort(new SortField("parser", new FieldCache.ShortParser() { - @Override - public short parseShort(BytesRef term) { - return (short) (term.bytes[term.offset]-'A'); - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - }), SortField.FIELD_DOC ); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - - // results should be in alphabetical order - assertEquals(10, td.totalHits); - Collections.sort(letters); - for (int i = 0; i < letters.size(); i++) { - assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); - } - - ir.close(); - dir.close(); - } - - /** - * test sorts for a custom long parser that uses a simple char encoding - */ - public void testCustomLongParser() throws Exception { - List letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); - Collections.shuffle(letters, random()); - - Directory dir = newDirectory(); - RandomIndexWriter iw = new RandomIndexWriter(random(), dir); - for (String letter : letters) { - Document doc = new Document(); - doc.add(newStringField("parser", letter, Field.Store.YES)); - iw.addDocument(doc); - } - - IndexReader ir = iw.getReader(); - iw.close(); - - IndexSearcher searcher = newSearcher(ir); - Sort sort = new Sort(new SortField("parser", new FieldCache.LongParser() { - @Override - public long parseLong(BytesRef term) { - return (term.bytes[term.offset]-'A') * 1234567890L; - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - }), SortField.FIELD_DOC ); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - - // results should be in alphabetical order - assertEquals(10, td.totalHits); - Collections.sort(letters); - for (int i = 0; i < letters.size(); i++) { - assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); - } - - ir.close(); - dir.close(); - } - - /** - * test sorts for a custom float parser that uses a simple char encoding - */ - public void testCustomFloatParser() throws Exception { - List letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); - Collections.shuffle(letters, random()); - - Directory dir = newDirectory(); - RandomIndexWriter iw = new RandomIndexWriter(random(), dir); - for (String letter : letters) { - Document doc = new Document(); - doc.add(newStringField("parser", letter, Field.Store.YES)); - iw.addDocument(doc); - } - - IndexReader ir = iw.getReader(); - iw.close(); - - IndexSearcher searcher = newSearcher(ir); - Sort sort = new Sort(new SortField("parser", new FieldCache.FloatParser() { - @Override - public float parseFloat(BytesRef term) { - return (float) Math.sqrt(term.bytes[term.offset]); - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - }), SortField.FIELD_DOC ); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - - // results should be in alphabetical order - assertEquals(10, td.totalHits); - Collections.sort(letters); - for (int i = 0; i < letters.size(); i++) { - assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); - } - - ir.close(); - dir.close(); - } - - /** - * test sorts for a custom double parser that uses a simple char encoding - */ - public void testCustomDoubleParser() throws Exception { - List letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); - Collections.shuffle(letters, random()); - - Directory dir = newDirectory(); - RandomIndexWriter iw = new RandomIndexWriter(random(), dir); - for (String letter : letters) { - Document doc = new Document(); - doc.add(newStringField("parser", letter, Field.Store.YES)); - iw.addDocument(doc); - } - - IndexReader ir = iw.getReader(); - iw.close(); - - IndexSearcher searcher = newSearcher(ir); - Sort sort = new Sort(new SortField("parser", new FieldCache.DoubleParser() { - @Override - public double parseDouble(BytesRef term) { - return Math.pow(term.bytes[term.offset], (term.bytes[term.offset]-'A')); - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - }), SortField.FIELD_DOC ); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - - // results should be in alphabetical order - assertEquals(10, td.totalHits); - Collections.sort(letters); - for (int i = 0; i < letters.size(); i++) { - assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); - } - - ir.close(); - dir.close(); - } - - /** Tests sorting a single document */ - public void testSortOneDocument() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "foo", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.STRING)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(1, td.totalHits); - assertEquals("foo", searcher.doc(td.scoreDocs[0].doc).get("value")); - - ir.close(); - dir.close(); - } - - /** Tests sorting a single document with scores */ - public void testSortOneDocumentWithScores() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("value", "foo", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.STRING)); - - TopDocs expected = searcher.search(new TermQuery(new Term("value", "foo")), 10); - assertEquals(1, expected.totalHits); - TopDocs actual = searcher.search(new TermQuery(new Term("value", "foo")), null, 10, sort, true, true); - - assertEquals(expected.totalHits, actual.totalHits); - assertEquals(expected.scoreDocs[0].score, actual.scoreDocs[0].score, 0F); - - ir.close(); - dir.close(); - } - - /** Tests sorting with two fields */ - public void testSortTwoFields() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(newStringField("tievalue", "tied", Field.Store.NO)); - doc.add(newStringField("value", "foo", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(newStringField("tievalue", "tied", Field.Store.NO)); - doc.add(newStringField("value", "bar", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = new IndexSearcher(ir); - // tievalue, then value - Sort sort = new Sort(new SortField("tievalue", SortField.Type.STRING), - new SortField("value", SortField.Type.STRING)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(2, td.totalHits); - // 'bar' comes before 'foo' - assertEquals("bar", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("foo", searcher.doc(td.scoreDocs[1].doc).get("value")); - - ir.close(); - dir.close(); - } -} From 3a4103d19b3445c4674dfa62c62d8139af02d15a Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Sat, 16 Feb 2013 01:27:57 +0000 Subject: [PATCH 04/13] SOLR-4459: The Replication 'index move' rather than copy optimization doesn't kick in when using NRTCachingDirectory or the rate limiting feature. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446833 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 6 +++- .../apache/solr/core/DirectoryFactory.java | 4 +++ .../solr/core/StandardDirectoryFactory.java | 36 +++++++++++++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index de7a966f71e..5040b74b530 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -107,7 +107,7 @@ Bug Fixes * SOLR-3926: Solr should support better way of finding active sorts (Eirik Lygre via Erick Erickson) -* SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman) +* SOLR-4342: Fix DataImportHandler stats to be a proper Map (hossman) * SOLR-3967: langid.enforceSchema option checks source field instead of target field (janhoy) @@ -125,6 +125,10 @@ Bug Fixes * SOLR-4463: Fix SolrCoreState reference counting. (Mark Miller) +* SOLR-4459: The Replication 'index move' rather than copy optimization doesn't + kick in when using NRTCachingDirectory or the rate limiting feature. + (Mark Miller) + Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java index 20507247bd1..3180ed225e6 100644 --- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java @@ -103,6 +103,10 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin, /** * Override for more efficient moves. * + * Intended for use with replication - use + * carefully - some Directory wrappers will + * cache files for example. + * * @throws IOException If there is a low-level I/O error. */ public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext) throws IOException { diff --git a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java index fd547a5074b..085cb8e0adc 100644 --- a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java @@ -23,6 +23,8 @@ import org.apache.commons.io.FileUtils; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.NRTCachingDirectory; +import org.apache.lucene.store.RateLimitedDirectoryWrapper; /** * Directory provider which mimics original Solr @@ -69,15 +71,29 @@ public class StandardDirectoryFactory extends CachingDirectoryFactory { /** * Override for more efficient moves. * + * Intended for use with replication - use + * carefully - some Directory wrappers will + * cache files for example. + * + * This implementation works with two wrappers: + * NRTCachingDirectory and RateLimitedDirectoryWrapper. + * + * You should first {@link Directory#sync(java.util.Collection)} any file that will be + * moved or avoid cached files through settings. + * * @throws IOException * If there is a low-level I/O error. */ @Override public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext) throws IOException { - if (fromDir instanceof FSDirectory && toDir instanceof FSDirectory) { - File dir1 = ((FSDirectory) fromDir).getDirectory(); - File dir2 = ((FSDirectory) toDir).getDirectory(); + + Directory baseFromDir = getBaseDir(fromDir); + Directory baseToDir = getBaseDir(fromDir); + + if (baseFromDir instanceof FSDirectory && baseToDir instanceof FSDirectory) { + File dir1 = ((FSDirectory) baseFromDir).getDirectory(); + File dir2 = ((FSDirectory) baseToDir).getDirectory(); File indexFileInTmpDir = new File(dir1, fileName); File indexFileInIndex = new File(dir2, fileName); boolean success = indexFileInTmpDir.renameTo(indexFileInIndex); @@ -89,4 +105,18 @@ public class StandardDirectoryFactory extends CachingDirectoryFactory { super.move(fromDir, toDir, fileName, ioContext); } + // special hack to work with NRTCachingDirectory and RateLimitedDirectoryWrapper + private Directory getBaseDir(Directory dir) { + Directory baseDir; + if (dir instanceof NRTCachingDirectory) { + baseDir = ((NRTCachingDirectory)dir).getDelegate(); + } else if (dir instanceof RateLimitedDirectoryWrapper) { + baseDir = ((RateLimitedDirectoryWrapper)dir).getDelegate(); + } else { + baseDir = dir; + } + + return baseDir; + } + } From 384d42b5e3be31514ca8c21dbfd706d82914e800 Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Sat, 16 Feb 2013 16:59:53 +0000 Subject: [PATCH 05/13] SOLR-4421,SOLR-4165: On CoreContainer shutdown, all SolrCores should publish their state as DOWN. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446914 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 6 +- .../org/apache/solr/cloud/ZkController.java | 143 +++++++++--------- .../org/apache/solr/core/CoreContainer.java | 28 ++-- .../solr/cloud/AbstractDistribZkTestBase.java | 2 +- 4 files changed, 90 insertions(+), 89 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 5040b74b530..40f546f080b 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -129,6 +129,9 @@ Bug Fixes kick in when using NRTCachingDirectory or the rate limiting feature. (Mark Miller) +* SOLR-4421,SOLR-4165: On CoreContainer shutdown, all SolrCores should publish their + state as DOWN. (Mark Miller, Markus Jelsma) + Optimizations ---------------------- @@ -161,9 +164,6 @@ Other Changes * SOLR-4384: Make post.jar report timing information (Upayavira via janhoy) -* SOLR-4421: On CoreContainer shutdown, all SolrCores should publish their - state as DOWN. (Mark Miller) - ================== 4.1.0 ================== Versions of Major Components diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 8b70a30ad16..f6ae41951f6 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -480,9 +480,14 @@ public final class ZkController { } private void init(CurrentCoreDescriptorProvider registerOnReconnect) { - boolean alreadyCreatedZkReader = false; + try { - alreadyCreatedZkReader = publishAndWaitForDownStates(alreadyCreatedZkReader); + boolean createdWatchesAndUpdated = false; + if (zkClient.exists(ZkStateReader.LIVE_NODES_ZKNODE, true)) { + zkStateReader.createClusterStateWatchersAndUpdate(); + createdWatchesAndUpdated = true; + publishAndWaitForDownStates(); + } // makes nodes zkNode cmdExecutor.ensureExists(ZkStateReader.LIVE_NODES_ZKNODE, zkClient); @@ -501,7 +506,7 @@ public final class ZkController { overseerElector.setup(context); overseerElector.joinElection(context, false); - if (!alreadyCreatedZkReader) { + if (!createdWatchesAndUpdated) { zkStateReader.createClusterStateWatchersAndUpdate(); } @@ -523,93 +528,92 @@ public final class ZkController { } - private boolean publishAndWaitForDownStates(boolean alreadyCreatedZkReader) - throws KeeperException, InterruptedException { - if (zkClient.exists(ZkStateReader.LIVE_NODES_ZKNODE, true)) { - alreadyCreatedZkReader = true; - // try and publish anyone from our node as down - zkStateReader.createClusterStateWatchersAndUpdate(); - ClusterState clusterState = zkStateReader.getClusterState(); - Set collections = clusterState.getCollections(); - List updatedNodes = new ArrayList(); + public void publishAndWaitForDownStates() throws KeeperException, + InterruptedException { + + ClusterState clusterState = zkStateReader.getClusterState(); + Set collections = clusterState.getCollections(); + List updatedNodes = new ArrayList(); + for (String collectionName : collections) { + DocCollection collection = clusterState.getCollection(collectionName); + Collection slices = collection.getSlices(); + for (Slice slice : slices) { + Collection replicas = slice.getReplicas(); + for (Replica replica : replicas) { + if (replica.getNodeName().equals(getNodeName()) + && !(replica.getStr(ZkStateReader.STATE_PROP) + .equals(ZkStateReader.DOWN))) { + ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state", + ZkStateReader.STATE_PROP, ZkStateReader.DOWN, + ZkStateReader.BASE_URL_PROP, getBaseUrl(), + ZkStateReader.CORE_NAME_PROP, + replica.getStr(ZkStateReader.CORE_NAME_PROP), + ZkStateReader.ROLES_PROP, + replica.getStr(ZkStateReader.ROLES_PROP), + ZkStateReader.NODE_NAME_PROP, getNodeName(), + ZkStateReader.SHARD_ID_PROP, + replica.getStr(ZkStateReader.SHARD_ID_PROP), + ZkStateReader.COLLECTION_PROP, + replica.getStr(ZkStateReader.COLLECTION_PROP)); + updatedNodes.add(replica.getStr(ZkStateReader.CORE_NAME_PROP)); + overseerJobQueue.offer(ZkStateReader.toJSON(m)); + } + } + } + } + + // now wait till the updates are in our state + long now = System.currentTimeMillis(); + long timeout = now + 1000 * 300; + boolean foundStates = false; + while (System.currentTimeMillis() < timeout) { + clusterState = zkStateReader.getClusterState(); + collections = clusterState.getCollections(); for (String collectionName : collections) { DocCollection collection = clusterState.getCollection(collectionName); Collection slices = collection.getSlices(); for (Slice slice : slices) { Collection replicas = slice.getReplicas(); for (Replica replica : replicas) { - if (replica.getNodeName().equals(getNodeName()) - && !(replica.getStr(ZkStateReader.STATE_PROP) - .equals(ZkStateReader.DOWN))) { - ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, - "state", ZkStateReader.STATE_PROP, ZkStateReader.DOWN, - ZkStateReader.BASE_URL_PROP, getBaseUrl(), - ZkStateReader.CORE_NAME_PROP, replica.getStr(ZkStateReader.CORE_NAME_PROP), - ZkStateReader.ROLES_PROP, - replica.getStr(ZkStateReader.ROLES_PROP), - ZkStateReader.NODE_NAME_PROP, getNodeName(), - ZkStateReader.SHARD_ID_PROP, - replica.getStr(ZkStateReader.SHARD_ID_PROP), - ZkStateReader.COLLECTION_PROP, - replica.getStr(ZkStateReader.COLLECTION_PROP)); - updatedNodes.add(replica.getStr(ZkStateReader.CORE_NAME_PROP)); - overseerJobQueue.offer(ZkStateReader.toJSON(m)); + if (replica.getStr(ZkStateReader.STATE_PROP).equals( + ZkStateReader.DOWN)) { + updatedNodes.remove(replica.getStr(ZkStateReader.CORE_NAME_PROP)); + } } } } - // now wait till the updates are in our state - long now = System.currentTimeMillis(); - long timeout = now + 1000 * 300; - boolean foundStates = false; - while (System.currentTimeMillis() < timeout) { - clusterState = zkStateReader.getClusterState(); - collections = clusterState.getCollections(); - for (String collectionName : collections) { - DocCollection collection = clusterState - .getCollection(collectionName); - Collection slices = collection.getSlices(); - for (Slice slice : slices) { - Collection replicas = slice.getReplicas(); - for (Replica replica : replicas) { - if (replica.getStr(ZkStateReader.STATE_PROP).equals( - ZkStateReader.DOWN)) { - updatedNodes.remove(replica - .getStr(ZkStateReader.CORE_NAME_PROP)); - - } - } - } - } - - if (updatedNodes.size() == 0) { - foundStates = true; - break; - } - } - if (!foundStates) { - log.warn("Timed out waiting to see all nodes published as DOWN in our cluster state."); + if (updatedNodes.size() == 0) { + foundStates = true; + break; } } - return alreadyCreatedZkReader; + if (!foundStates) { + log.warn("Timed out waiting to see all nodes published as DOWN in our cluster state."); + } + } - + /** - * Validates if the chroot exists in zk (or if it is successfully created). Optionally, if create is set to true this method will create the path - * in case it doesn't exist - * @return true if the path exists or is created - * false if the path doesn't exist and 'create' = false + * Validates if the chroot exists in zk (or if it is successfully created). + * Optionally, if create is set to true this method will create the path in + * case it doesn't exist + * + * @return true if the path exists or is created false if the path doesn't + * exist and 'create' = false */ - public static boolean checkChrootPath(String zkHost, boolean create) throws KeeperException, InterruptedException { - if(!containsChroot(zkHost)) { + public static boolean checkChrootPath(String zkHost, boolean create) + throws KeeperException, InterruptedException { + if (!containsChroot(zkHost)) { return true; } log.info("zkHost includes chroot"); String chrootPath = zkHost.substring(zkHost.indexOf("/"), zkHost.length()); - SolrZkClient tmpClient = new SolrZkClient(zkHost.substring(0, zkHost.indexOf("/")), 60*1000); + SolrZkClient tmpClient = new SolrZkClient(zkHost.substring(0, + zkHost.indexOf("/")), 60 * 1000); boolean exists = tmpClient.exists(chrootPath, true); - if(!exists && create) { + if (!exists && create) { tmpClient.makePath(chrootPath, false, true); exists = true; } @@ -617,7 +621,6 @@ public final class ZkController { return exists; } - /** * Validates if zkHost contains a chroot. See http://zookeeper.apache.org/doc/r3.2.2/zookeeperProgrammers.html#ch_zkSessions */ diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 44dd5a79c9a..1f21d45031a 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -740,12 +740,24 @@ public class CoreContainer public void shutdown() { log.info("Shutting down CoreContainer instance=" + System.identityHashCode(this)); + + if (isZooKeeperAware()) { + try { + zkController.publishAndWaitForDownStates(); + } catch (KeeperException e) { + log.error("", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.warn("", e); + } + } + isShutDown = true; if (isZooKeeperAware()) { - publishCoresAsDown(); cancelCoreRecoveries(); } + try { synchronized (cores) { @@ -784,20 +796,6 @@ public class CoreContainer } } - private void publishCoresAsDown() { - synchronized (cores) { - for (SolrCore core : cores.values()) { - try { - zkController.publish(core.getCoreDescriptor(), ZkStateReader.DOWN); - } catch (KeeperException e) { - log.error("", e); - } catch (InterruptedException e) { - log.error("", e); - } - } - } - } - public void cancelCoreRecoveries() { ArrayList coreStates = new ArrayList(); synchronized (cores) { diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java index c5ff64be114..f28c5989fda 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java @@ -207,7 +207,6 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes if (DEBUG) { printLayout(); } - zkServer.shutdown(); System.clearProperty("zkHost"); System.clearProperty("collection"); System.clearProperty("enable.update.log"); @@ -217,6 +216,7 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes System.clearProperty("solr.test.sys.prop2"); resetExceptionIgnores(); super.tearDown(); + zkServer.shutdown(); } protected void printLayout() throws Exception { From e61398084d3f1ca0f28c5c35d3318645d7a401ec Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Sat, 16 Feb 2013 18:50:20 +0000 Subject: [PATCH 06/13] SOLR-3855: Doc values support. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446922 13f79535-47bb-0310-9956-ffa450edef68 --- .../function/valuesource/IntFieldSource.java | 2 +- .../function/valuesource/LongFieldSource.java | 9 + solr/CHANGES.txt | 2 + .../apache/solr/core/SchemaCodecFactory.java | 14 +- .../java/org/apache/solr/core/SolrCore.java | 5 + .../handler/component/FieldFacetStats.java | 124 ++++--- .../handler/component/StatsComponent.java | 82 ++--- .../solr/handler/component/StatsValues.java | 17 +- .../handler/component/StatsValuesFactory.java | 96 ++--- .../apache/solr/request/NumericFacets.java | 328 ++++++++++++++++++ .../org/apache/solr/request/SimpleFacets.java | 115 ++++-- .../apache/solr/request/UnInvertedField.java | 8 +- .../solr/schema/AbstractSpatialFieldType.java | 28 +- .../org/apache/solr/schema/CurrencyField.java | 12 +- .../org/apache/solr/schema/DateField.java | 15 +- .../apache/solr/schema/FieldProperties.java | 3 +- .../org/apache/solr/schema/FieldType.java | 70 ++-- .../org/apache/solr/schema/LatLonType.java | 60 ++-- .../org/apache/solr/schema/PointType.java | 8 +- .../org/apache/solr/schema/SchemaField.java | 15 +- .../solr/schema/SortableDoubleField.java | 13 +- .../solr/schema/SortableFloatField.java | 13 +- .../apache/solr/schema/SortableIntField.java | 13 +- .../apache/solr/schema/SortableLongField.java | 13 +- .../java/org/apache/solr/schema/StrField.java | 45 ++- .../org/apache/solr/schema/TrieDateField.java | 22 +- .../org/apache/solr/schema/TrieField.java | 69 +++- .../org/apache/solr/schema/UUIDField.java | 2 - .../apache/solr/update/DocumentBuilder.java | 47 +-- ...hema-docValues-not-required-no-default.xml | 33 ++ .../conf/bad-schema-unsupported-docValues.xml | 30 ++ .../collection1/conf/schema-docValues.xml | 74 ++++ .../solr/collection1/conf/schema.xml | 2 +- .../solr/collection1/conf/schema_codec.xml | 19 +- .../apache/solr/core/TestCodecSupport.java | 27 +- .../handler/component/StatsComponentTest.java | 8 + .../solr/schema/BadIndexSchemaTest.java | 9 +- .../apache/solr/schema/CurrencyFieldTest.java | 15 +- .../org/apache/solr/schema/DocValuesTest.java | 230 ++++++++++++ .../org/apache/solr/schema/PolyFieldTest.java | 17 +- solr/example/solr/collection1/conf/schema.xml | 28 +- 41 files changed, 1336 insertions(+), 406 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/request/NumericFacets.java create mode 100644 solr/core/src/test-files/solr/collection1/conf/bad-schema-docValues-not-required-no-default.xml create mode 100644 solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-docValues.xml create mode 100644 solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml create mode 100644 solr/core/src/test/org/apache/solr/schema/DocValuesTest.java diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java index c8a9a9af82c..296432d6f9a 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java @@ -85,7 +85,7 @@ public class IntFieldSource extends FieldCacheSource { @Override public String strVal(int doc) { - return Float.toString(arr.get(doc)); + return Integer.toString(arr.get(doc)); } @Override diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java index 1a8a9ad666d..597efe89e97 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java @@ -64,6 +64,10 @@ public class LongFieldSource extends FieldCacheSource { return val; } + public String longToString(long val) { + return longToObject(val).toString(); + } + @Override public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { final FieldCache.Longs arr = cache.getLongs(readerContext.reader(), field, parser, true); @@ -85,6 +89,11 @@ public class LongFieldSource extends FieldCacheSource { return valid.get(doc) ? longToObject(arr.get(doc)) : null; } + @Override + public String strVal(int doc) { + return valid.get(doc) ? longToString(arr.get(doc)) : null; + } + @Override public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) { long lower,upper; diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 40f546f080b..d908578cd01 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -77,6 +77,8 @@ New Features under the covers -- allowing many HTTP connection related properties to be controlled via 'standard' java system properties. (hossman) +* SOLR-3855: Doc values support. (Adrien Grand) + Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java index 5e5a81d4e4b..e075913066a 100644 --- a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java +++ b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java @@ -1,6 +1,7 @@ package org.apache.solr.core; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene42.Lucene42Codec; import org.apache.solr.schema.IndexSchema; @@ -55,7 +56,18 @@ public class SchemaCodecFactory extends CodecFactory implements SchemaAware { } return super.getPostingsFormatForField(field); } - // TODO: when dv support is added to solr, add it here too + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + final SchemaField fieldOrNull = schema.getFieldOrNull(field); + if (fieldOrNull == null) { + throw new IllegalArgumentException("no such field " + field); + } + String docValuesFormatName = fieldOrNull.getType().getDocValuesFormat(); + if (docValuesFormatName != null) { + return DocValuesFormat.forName(docValuesFormatName); + } + return super.getDocValuesFormatForField(field); + } }; } diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index a3dd8f9af9a..4abdb6855c5 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -834,6 +834,11 @@ public final class SolrCore implements SolrInfoMBean { log.error(msg); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg); } + if (null != ft.getDocValuesFormat()) { + String msg = "FieldType '" + ft.getTypeName() + "' is configured with a docValues format, but the codec does not support it: " + factory.getClass(); + log.error(msg); + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg); + } } } return factory.getCodec(); diff --git a/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java b/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java index adce22e90d2..6cd9a190573 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java +++ b/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java @@ -16,16 +16,22 @@ package org.apache.solr.handler.component; * limitations under the License. */ +import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.FieldCache; import org.apache.lucene.util.BytesRef; -import org.apache.solr.schema.FieldType; import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.SolrIndexSearcher; /** @@ -40,101 +46,76 @@ import org.apache.solr.schema.SchemaField; public class FieldFacetStats { public final String name; - final SortedDocValues si; final SchemaField facet_sf; final SchemaField field_sf; - final int startTermIndex; - final int endTermIndex; - final int nTerms; - - final int numStatsTerms; - public final Map facetStatsValues; - final List> facetStatsTerms; + List> facetStatsTerms; + + final AtomicReader topLevelReader; + AtomicReaderContext leave; + final ValueSource valueSource; + AtomicReaderContext context; + FunctionValues values; + + SortedDocValues topLevelSortedValues = null; private final BytesRef tempBR = new BytesRef(); - public FieldFacetStats(String name, SortedDocValues si, SchemaField field_sf, SchemaField facet_sf, int numStatsTerms) { + public FieldFacetStats(SolrIndexSearcher searcher, String name, SchemaField field_sf, SchemaField facet_sf) { this.name = name; - this.si = si; this.field_sf = field_sf; this.facet_sf = facet_sf; - this.numStatsTerms = numStatsTerms; - startTermIndex = 0; - endTermIndex = si.getValueCount(); - nTerms = endTermIndex - startTermIndex; + topLevelReader = searcher.getAtomicReader(); + valueSource = facet_sf.getType().getValueSource(facet_sf, null); facetStatsValues = new HashMap(); - - // for mv stats field, we'll want to keep track of terms facetStatsTerms = new ArrayList>(); - if (numStatsTerms == 0) return; - int i = 0; - for (; i < numStatsTerms; i++) { - facetStatsTerms.add(new HashMap()); - } } - BytesRef getTermText(int docID, BytesRef ret) { - final int ord = si.getOrd(docID); - if (ord == -1) { - return null; - } else { - si.lookupOrd(ord, ret); - return ret; + private StatsValues getStatsValues(String key) throws IOException { + StatsValues stats = facetStatsValues.get(key); + if (stats == null) { + stats = StatsValuesFactory.createStatsValues(field_sf); + facetStatsValues.put(key, stats); + stats.setNextReader(context); } + return stats; } - public boolean facet(int docID, BytesRef v) { - int term = si.getOrd(docID); - int arrIdx = term - startTermIndex; - if (arrIdx >= 0 && arrIdx < nTerms) { - - final BytesRef br; - if (term == -1) { - br = null; - } else { - br = tempBR; - si.lookupOrd(term, tempBR); - } - String key = (br == null)?null:facet_sf.getType().indexedToReadable(br.utf8ToString()); - StatsValues stats = facetStatsValues.get(key); - if (stats == null) { - stats = StatsValuesFactory.createStatsValues(field_sf); - facetStatsValues.put(key, stats); - } - - if (v != null && v.length>0) { - stats.accumulate(v); - } else { - stats.missing(); - return false; - } - return true; - } - return false; + // docID is relative to the context + public void facet(int docID) throws IOException { + final String key = values.exists(docID) + ? values.strVal(docID) + : null; + final StatsValues stats = getStatsValues(key); + stats.accumulate(docID); } - // Function to keep track of facet counts for term number. // Currently only used by UnInvertedField stats - public boolean facetTermNum(int docID, int statsTermNum) { - - int term = si.getOrd(docID); - int arrIdx = term - startTermIndex; - if (arrIdx >= 0 && arrIdx < nTerms) { + public boolean facetTermNum(int docID, int statsTermNum) throws IOException { + if (topLevelSortedValues == null) { + topLevelSortedValues = FieldCache.DEFAULT.getTermsIndex(topLevelReader, name); + } + + int term = topLevelSortedValues.getOrd(docID); + int arrIdx = term; + if (arrIdx >= 0 && arrIdx < topLevelSortedValues.getValueCount()) { final BytesRef br; if (term == -1) { br = null; } else { br = tempBR; - si.lookupOrd(term, tempBR); + topLevelSortedValues.lookupOrd(term, tempBR); } String key = br == null ? null : br.utf8ToString(); - HashMap statsTermCounts = facetStatsTerms.get(statsTermNum); + while (facetStatsTerms.size() <= statsTermNum) { + facetStatsTerms.add(new HashMap()); + } + final Map statsTermCounts = facetStatsTerms.get(statsTermNum); Integer statsTermCount = statsTermCounts.get(key); if (statsTermCount == null) { statsTermCounts.put(key, 1); @@ -148,8 +129,11 @@ public class FieldFacetStats { //function to accumulate counts for statsTermNum to specified value - public boolean accumulateTermNum(int statsTermNum, BytesRef value) { + public boolean accumulateTermNum(int statsTermNum, BytesRef value) throws IOException { if (value == null) return false; + while (facetStatsTerms.size() <= statsTermNum) { + facetStatsTerms.add(new HashMap()); + } for (Map.Entry stringIntegerEntry : facetStatsTerms.get(statsTermNum).entrySet()) { Map.Entry pairs = (Map.Entry) stringIntegerEntry; String key = (String) pairs.getKey(); @@ -166,6 +150,14 @@ public class FieldFacetStats { return true; } + public void setNextReader(AtomicReaderContext ctx) throws IOException { + this.context = ctx; + values = valueSource.getValues(Collections.emptyMap(), ctx); + for (StatsValues stats : facetStatsValues.values()) { + stats.setNextReader(ctx); + } + } + } diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java b/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java index 521dc832b5e..dc433b42783 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java @@ -20,12 +20,11 @@ package org.apache.solr.handler.component; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.search.FieldCache; -import org.apache.lucene.util.BytesRef; +import org.apache.lucene.index.AtomicReaderContext; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; @@ -43,14 +42,12 @@ import org.apache.solr.search.SolrIndexSearcher; /** * Stats component calculates simple statistics on numeric field values - * - * * @since solr 1.4 */ public class StatsComponent extends SearchComponent { public static final String COMPONENT_NAME = "stats"; - + @Override public void prepare(ResponseBuilder rb) throws IOException { if (rb.req.getParams().getBool(StatsParams.STATS,false)) { @@ -236,25 +233,13 @@ class SimpleStats { } return res; } - - // why does this use a top-level field cache? - public NamedList getFieldCacheStats(String fieldName, String[] facet ) { - SchemaField sf = searcher.getSchema().getField(fieldName); - - SortedDocValues si; - try { - si = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), fieldName); - } - catch (IOException e) { - throw new RuntimeException( "failed to open field cache for: "+fieldName, e ); - } - StatsValues allstats = StatsValuesFactory.createStatsValues(sf); - final int nTerms = si.getValueCount(); - if ( nTerms <= 0 || docs.size() <= 0 ) return allstats.getStatsValues(); - // don't worry about faceting if no documents match... + public NamedList getFieldCacheStats(String fieldName, String[] facet) throws IOException { + final SchemaField sf = searcher.getSchema().getField(fieldName); + + final StatsValues allstats = StatsValuesFactory.createStatsValues(sf); + List facetStats = new ArrayList(); - SortedDocValues facetTermsIndex; for( String facetField : facet ) { SchemaField fsf = searcher.getSchema().getField(facetField); @@ -262,40 +247,32 @@ class SimpleStats { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Stats can only facet on single-valued fields, not: " + facetField ); } - - try { - facetTermsIndex = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), facetField); - } - catch (IOException e) { - throw new RuntimeException( "failed to open field cache for: " - + facetField, e ); - } - facetStats.add(new FieldFacetStats(facetField, facetTermsIndex, sf, fsf, nTerms)); + + facetStats.add(new FieldFacetStats(searcher, facetField, sf, fsf)); } - - final BytesRef tempBR = new BytesRef(); - DocIterator iter = docs.iterator(); - while (iter.hasNext()) { - int docID = iter.nextDoc(); - int docOrd = si.getOrd(docID); - BytesRef raw; - if (docOrd == -1) { - allstats.missing(); - tempBR.length = 0; - raw = tempBR; - } else { - raw = tempBR; - si.lookupOrd(docOrd, tempBR); - if( tempBR.length > 0 ) { - allstats.accumulate(tempBR); - } else { - allstats.missing(); + + final Iterator ctxIt = searcher.getIndexReader().leaves().iterator(); + AtomicReaderContext ctx = null; + for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) { + final int doc = docsIt.nextDoc(); + if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) { + // advance + do { + ctx = ctxIt.next(); + } while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()); + assert doc >= ctx.docBase; + + // propagate the context among accumulators. + allstats.setNextReader(ctx); + for (FieldFacetStats f : facetStats) { + f.setNextReader(ctx); } } - // now update the facets + // accumulate + allstats.accumulate(doc - ctx.docBase); for (FieldFacetStats f : facetStats) { - f.facet(docID, raw); + f.facet(doc - ctx.docBase); } } @@ -305,5 +282,4 @@ class SimpleStats { return allstats.getStatsValues(); } - } diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java index 492ef0148b6..cbcde045ea1 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java @@ -19,14 +19,19 @@ package org.apache.solr.handler.component; +import java.io.IOException; import java.util.Map; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.util.BytesRef; import org.apache.solr.common.util.NamedList; +import org.apache.solr.schema.FieldType; /** * StatsValue defines the interface for the collection of statistical values about fields and facets. */ +// TODO: should implement Collector? public interface StatsValues { /** @@ -36,12 +41,9 @@ public interface StatsValues { */ void accumulate(NamedList stv); - /** - * Accumulate the values based on the given value - * - * @param value Value to use to accumulate the current values - */ - void accumulate(BytesRef value); + /** Accumulate the value associated with docID. + * @see #setNextReader(AtomicReaderContext) */ + void accumulate(int docID); /** * Accumulate the values based on the given value @@ -77,4 +79,7 @@ public interface StatsValues { * @return NamedList representation of the current values */ NamedList getStatsValues(); + + /** Set the context for {@link #accumulate(int)}. */ + void setNextReader(AtomicReaderContext ctx) throws IOException; } diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java index d4ef1c5831e..c350dd5d936 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java @@ -17,10 +17,15 @@ package org.apache.solr.handler.component; +import java.io.IOException; +import java.util.Collections; import java.util.Date; import java.util.Map; import java.util.HashMap; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.NamedList; @@ -39,6 +44,7 @@ public class StatsValuesFactory { * @return Instance of StatsValues that will create statistics from values from a field of the given type */ public static StatsValues createStatsValues(SchemaField sf) { + // TODO: allow for custom field types FieldType fieldType = sf.getType(); if (DoubleField.class.isInstance(fieldType) || IntField.class.isInstance(fieldType) || @@ -77,6 +83,8 @@ abstract class AbstractStatsValues implements StatsValues { protected T min; protected long missing; protected long count; + private ValueSource valueSource; + protected FunctionValues values; // facetField facetValue protected Map> facets = new HashMap>(); @@ -121,29 +129,22 @@ abstract class AbstractStatsValues implements StatsValues { } } } - - /** - * {@inheritDoc} - */ - @Override - public void accumulate(BytesRef value) { - count++; - T typedValue = (T)ft.toObject(sf, value); - updateMinMax(typedValue, typedValue); - updateTypeSpecificStats(typedValue); - } /** * {@inheritDoc} */ @Override public void accumulate(BytesRef value, int count) { - this.count += count; T typedValue = (T)ft.toObject(sf, value); - updateMinMax(typedValue, typedValue); - updateTypeSpecificStats(typedValue, count); + accumulate(typedValue, count); } - + + public void accumulate(T value, int count) { + this.count += count; + updateMinMax(value, value); + updateTypeSpecificStats(value, count); + } + /** * {@inheritDoc} */ @@ -194,6 +195,13 @@ abstract class AbstractStatsValues implements StatsValues { return res; } + public void setNextReader(AtomicReaderContext ctx) throws IOException { + if (valueSource == null) { + valueSource = ft.getValueSource(sf, null); + } + values = valueSource.getValues(Collections.emptyMap(), ctx); + } + /** * Updates the minimum and maximum statistics based on the given values * @@ -202,13 +210,6 @@ abstract class AbstractStatsValues implements StatsValues { */ protected abstract void updateMinMax(T min, T max); - /** - * Updates the type specific statistics based on the given value - * - * @param value Value the statistics should be updated against - */ - protected abstract void updateTypeSpecificStats(T value); - /** * Updates the type specific statistics based on the given value * @@ -246,6 +247,15 @@ class NumericStatsValues extends AbstractStatsValues { max = Double.NEGATIVE_INFINITY; } + @Override + public void accumulate(int docID) { + if (values.exists(docID)) { + accumulate((Number) values.objectVal(docID), 1); + } else { + missing(); + } + } + /** * {@inheritDoc} */ @@ -255,16 +265,6 @@ class NumericStatsValues extends AbstractStatsValues { sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue(); } - /** - * {@inheritDoc} - */ - @Override - public void updateTypeSpecificStats(Number v) { - double value = v.doubleValue(); - sumOfSquares += (value * value); // for std deviation - sum += value; - } - /** * {@inheritDoc} */ @@ -323,6 +323,15 @@ class DateStatsValues extends AbstractStatsValues { super(sf); } + @Override + public void accumulate(int docID) { + if (values.exists(docID)) { + accumulate((Date) values.objectVal(docID), 1); + } else { + missing(); + } + } + /** * {@inheritDoc} */ @@ -332,16 +341,6 @@ class DateStatsValues extends AbstractStatsValues { sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue(); } - /** - * {@inheritDoc} - */ - @Override - public void updateTypeSpecificStats(Date v) { - long value = v.getTime(); - sumOfSquares += (value * value); // for std deviation - sum += value; - } - /** * {@inheritDoc} */ @@ -407,19 +406,20 @@ class StringStatsValues extends AbstractStatsValues { super(sf); } - /** - * {@inheritDoc} - */ @Override - protected void updateTypeSpecificStats(NamedList stv) { - // No type specific stats + public void accumulate(int docID) { + if (values.exists(docID)) { + accumulate(values.strVal(docID), 1); + } else { + missing(); + } } /** * {@inheritDoc} */ @Override - protected void updateTypeSpecificStats(String value) { + protected void updateTypeSpecificStats(NamedList stv) { // No type specific stats } diff --git a/solr/core/src/java/org/apache/solr/request/NumericFacets.java b/solr/core/src/java/org/apache/solr/request/NumericFacets.java new file mode 100644 index 00000000000..7a6ec4c9861 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/request/NumericFacets.java @@ -0,0 +1,328 @@ +package org.apache.solr.request; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.document.FieldType.NumericType; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.ReaderUtil; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRef; +import org.apache.lucene.util.PriorityQueue; +import org.apache.solr.common.params.FacetParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocSet; +import org.apache.solr.search.SolrIndexSearcher; + +/** Utility class to compute facets on numeric fields. */ +final class NumericFacets { + + NumericFacets() {} + + static class HashTable { + + static final float LOAD_FACTOR = 0.7f; + + long[] bits; // bits identifying a value + int[] counts; + int[] docIDs; + int mask; + int size; + int threshold; + + HashTable() { + final int capacity = 64; // must be a power of 2 + bits = new long[capacity]; + counts = new int[capacity]; + docIDs = new int[capacity]; + mask = capacity - 1; + size = 0; + threshold = (int) (capacity * LOAD_FACTOR); + } + + private int hash(long v) { + int h = (int) (v ^ (v >>> 32)); + h = (31 * h) & mask; // * 31 to try to use the whole table, even if values are dense + return h; + } + + void add(int docID, long value, int count) { + if (size >= threshold) { + rehash(); + } + final int h = hash(value); + for (int slot = h; ; slot = (slot + 1) & mask) { + if (counts[slot] == 0) { + bits[slot] = value; + docIDs[slot] = docID; + ++size; + } else if (bits[slot] != value) { + continue; + } + counts[slot] += count; + break; + } + } + + private void rehash() { + final long[] oldBits = bits; + final int[] oldCounts = counts; + final int[] oldDocIDs = docIDs; + + final int newCapacity = bits.length * 2; + bits = new long[newCapacity]; + counts = new int[newCapacity]; + docIDs = new int[newCapacity]; + mask = newCapacity - 1; + threshold = (int) (LOAD_FACTOR * newCapacity); + size = 0; + + for (int i = 0; i < oldBits.length; ++i) { + if (oldCounts[i] > 0) { + add(oldDocIDs[i], oldBits[i], oldCounts[i]); + } + } + } + + } + + private static class Entry { + int docID; + int count; + long bits; + } + + public static NamedList getCounts(SolrIndexSearcher searcher, DocSet docs, String fieldName, int offset, int limit, int mincount, boolean missing, String sort) throws IOException { + final boolean zeros = mincount <= 0; + mincount = Math.max(mincount, 1); + final SchemaField sf = searcher.getSchema().getField(fieldName); + final FieldType ft = sf.getType(); + final NumericType numericType = ft.getNumericType(); + if (numericType == null) { + throw new IllegalStateException(); + } + final List leaves = searcher.getIndexReader().leaves(); + + // 1. accumulate + final HashTable hashTable = new HashTable(); + final Iterator ctxIt = leaves.iterator(); + AtomicReaderContext ctx = null; + FieldCache.Longs longs = null; + Bits docsWithField = null; + int missingCount = 0; + for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) { + final int doc = docsIt.nextDoc(); + if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) { + do { + ctx = ctxIt.next(); + } while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()); + assert doc >= ctx.docBase; + switch (numericType) { + case LONG: + longs = FieldCache.DEFAULT.getLongs(ctx.reader(), fieldName, true); + break; + case INT: + final FieldCache.Ints ints = FieldCache.DEFAULT.getInts(ctx.reader(), fieldName, true); + longs = new FieldCache.Longs() { + @Override + public long get(int docID) { + return ints.get(docID); + } + }; + break; + case FLOAT: + final FieldCache.Floats floats = FieldCache.DEFAULT.getFloats(ctx.reader(), fieldName, true); + longs = new FieldCache.Longs() { + @Override + public long get(int docID) { + return Float.floatToIntBits(floats.get(docID)); + } + }; + break; + case DOUBLE: + final FieldCache.Doubles doubles = FieldCache.DEFAULT.getDoubles(ctx.reader(), fieldName, true); + longs = new FieldCache.Longs() { + @Override + public long get(int docID) { + return Double.doubleToLongBits(doubles.get(docID)); + } + }; + break; + default: + throw new AssertionError(); + } + docsWithField = FieldCache.DEFAULT.getDocsWithField(ctx.reader(), fieldName); + } + if (docsWithField.get(doc - ctx.docBase)) { + hashTable.add(doc, longs.get(doc - ctx.docBase), 1); + } else { + ++missingCount; + } + } + + // 2. select top-k facet values + final int pqSize = limit < 0 ? hashTable.size : Math.min(offset + limit, hashTable.size); + final PriorityQueue pq; + if (FacetParams.FACET_SORT_COUNT.equals(sort) || FacetParams.FACET_SORT_COUNT_LEGACY.equals(sort)) { + pq = new PriorityQueue(pqSize) { + @Override + protected boolean lessThan(Entry a, Entry b) { + if (a.count < b.count || (a.count == b.count && a.bits > b.bits)) { + return true; + } else { + return false; + } + } + }; + } else { + pq = new PriorityQueue(pqSize) { + @Override + protected boolean lessThan(Entry a, Entry b) { + return a.bits > b.bits; + } + }; + } + Entry e = null; + for (int i = 0; i < hashTable.bits.length; ++i) { + if (hashTable.counts[i] >= mincount) { + if (e == null) { + e = new Entry(); + } + e.bits = hashTable.bits[i]; + e.count = hashTable.counts[i]; + e.docID = hashTable.docIDs[i]; + e = pq.insertWithOverflow(e); + } + } + + // 4. build the NamedList + final ValueSource vs = ft.getValueSource(sf, null); + final NamedList result = new NamedList(); + + // This stuff is complicated because if facet.mincount=0, the counts needs + // to be merged with terms from the terms dict + if (!zeros || FacetParams.FACET_SORT_COUNT.equals(sort) || FacetParams.FACET_SORT_COUNT_LEGACY.equals(sort)) { + // Only keep items we're interested in + final Deque counts = new ArrayDeque(); + while (pq.size() > offset) { + counts.addFirst(pq.pop()); + } + + // Entries from the PQ first, then using the terms dictionary + for (Entry entry : counts) { + final int readerIdx = ReaderUtil.subIndex(entry.docID, leaves); + final FunctionValues values = vs.getValues(Collections.emptyMap(), leaves.get(readerIdx)); + result.add(values.strVal(entry.docID - leaves.get(readerIdx).docBase), entry.count); + } + + if (zeros && (limit < 0 || result.size() < limit)) { // need to merge with the term dict + if (!sf.indexed()) { + throw new IllegalStateException("Cannot use " + FacetParams.FACET_MINCOUNT + "=0 on a field which is not indexed"); + } + // Add zeros until there are limit results + final Set alreadySeen = new HashSet(); + while (pq.size() > 0) { + Entry entry = pq.pop(); + final int readerIdx = ReaderUtil.subIndex(entry.docID, leaves); + final FunctionValues values = vs.getValues(Collections.emptyMap(), leaves.get(readerIdx)); + alreadySeen.add(values.strVal(entry.docID - leaves.get(readerIdx).docBase)); + } + for (int i = 0; i < result.size(); ++i) { + alreadySeen.add(result.getName(i)); + } + final Terms terms = searcher.getAtomicReader().terms(fieldName); + if (terms != null) { + final TermsEnum termsEnum = terms.iterator(null); + BytesRef term = termsEnum.next(); + final CharsRef spare = new CharsRef(); + for (int skipped = hashTable.size; skipped < offset && term != null; ) { + ft.indexedToReadable(term, spare); + final String termStr = spare.toString(); + if (!alreadySeen.contains(termStr)) { + ++skipped; + } + term = termsEnum.next(); + } + for ( ; term != null && (limit < 0 || result.size() < limit); term = termsEnum.next()) { + ft.indexedToReadable(term, spare); + final String termStr = spare.toString(); + if (!alreadySeen.contains(termStr)) { + result.add(termStr, 0); + } + } + } + } + } else { + // sort=index, mincount=0 and we have less than limit items + // => Merge the PQ and the terms dictionary on the fly + if (!sf.indexed()) { + throw new IllegalStateException("Cannot use " + FacetParams.FACET_SORT + "=" + FacetParams.FACET_SORT_INDEX + " on a field which is not indexed"); + } + final Map counts = new HashMap(); + while (pq.size() > 0) { + final Entry entry = pq.pop(); + final int readerIdx = ReaderUtil.subIndex(entry.docID, leaves); + final FunctionValues values = vs.getValues(Collections.emptyMap(), leaves.get(readerIdx)); + counts.put(values.strVal(entry.docID - leaves.get(readerIdx).docBase), entry.count); + } + final Terms terms = searcher.getAtomicReader().terms(fieldName); + if (terms != null) { + final TermsEnum termsEnum = terms.iterator(null); + final CharsRef spare = new CharsRef(); + BytesRef term = termsEnum.next(); + for (int i = 0; i < offset && term != null; ++i) { + term = termsEnum.next(); + } + for ( ; term != null && (limit < 0 || result.size() < limit); term = termsEnum.next()) { + ft.indexedToReadable(term, spare); + final String termStr = spare.toString(); + Integer count = counts.get(termStr); + if (count == null) { + count = 0; + } + result.add(termStr, count); + } + } + } + + if (missing) { + result.add(null, missingCount); + } + return result; + } + +} diff --git a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java index 5dbd054a7a7..f0f48787b14 100644 --- a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java +++ b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java @@ -17,37 +17,83 @@ package org.apache.solr.request; -import org.apache.lucene.index.*; -import org.apache.lucene.search.*; -import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector; -import org.apache.lucene.search.grouping.term.TermGroupFacetCollector; -import org.apache.lucene.search.grouping.term.TermAllGroupsCollector; -import org.apache.lucene.util.*; -import org.apache.lucene.util.packed.PackedInts; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.params.*; -import org.apache.solr.common.params.FacetParams.FacetRangeOther; -import org.apache.solr.common.params.FacetParams.FacetRangeInclude; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.schema.*; -import org.apache.solr.search.*; -import org.apache.solr.search.grouping.GroupingSpecification; -import org.apache.solr.util.BoundedTreeSet; -import org.apache.solr.util.DateMathParser; -import org.apache.solr.util.DefaultSolrThreadFactory; -import org.apache.solr.handler.component.ResponseBuilder; -import org.apache.solr.util.LongPriorityQueue; - import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.EnumSet; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.MultiDocsEnum; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector; +import org.apache.lucene.search.grouping.term.TermAllGroupsCollector; +import org.apache.lucene.search.grouping.term.TermGroupFacetCollector; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRef; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.StringHelper; +import org.apache.lucene.util.UnicodeUtil; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.FacetParams; +import org.apache.solr.common.params.FacetParams.FacetRangeInclude; +import org.apache.solr.common.params.FacetParams.FacetRangeOther; +import org.apache.solr.common.params.GroupParams; +import org.apache.solr.common.params.RequiredSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.schema.BoolField; +import org.apache.solr.schema.DateField; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.SortableDoubleField; +import org.apache.solr.schema.SortableFloatField; +import org.apache.solr.schema.SortableIntField; +import org.apache.solr.schema.SortableLongField; +import org.apache.solr.schema.TrieField; +import org.apache.solr.search.BitDocSet; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocSet; +import org.apache.solr.search.Grouping; +import org.apache.solr.search.HashDocSet; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SortedIntDocSet; +import org.apache.solr.search.SyntaxError; +import org.apache.solr.search.grouping.GroupingSpecification; +import org.apache.solr.util.BoundedTreeSet; +import org.apache.solr.util.DateMathParser; +import org.apache.solr.util.DefaultSolrThreadFactory; +import org.apache.solr.util.LongPriorityQueue; + /** * A class that generates simple Facet information for a request. * @@ -300,7 +346,8 @@ public class SimpleFacets { boolean enumMethod = FacetParams.FACET_METHOD_enum.equals(method); // TODO: default to per-segment or not? - boolean per_segment = FacetParams.FACET_METHOD_fcs.equals(method); + boolean per_segment = FacetParams.FACET_METHOD_fcs.equals(method) // explicit + || (ft.getNumericType() != null && sf.hasDocValues()); // numeric doc values are per-segment by default if (method == null && ft instanceof BoolField) { // Always use filters for booleans... we know the number of values is very small. @@ -329,10 +376,18 @@ public class SimpleFacets { // TODO: future logic could use filters instead of the fieldcache if // the number of terms in the field is small enough. if (per_segment) { - PerSegmentSingleValuedFaceting ps = new PerSegmentSingleValuedFaceting(searcher, docs, field, offset,limit, mincount, missing, sort, prefix); - Executor executor = threads == 0 ? directExecutor : facetExecutor; - ps.setNumThreads(threads); - counts = ps.getFacetCounts(executor); + if (ft.getNumericType() != null && !sf.multiValued()) { + // force numeric faceting + if (prefix != null && !prefix.isEmpty()) { + throw new SolrException(ErrorCode.BAD_REQUEST, FacetParams.FACET_PREFIX + " is not supported on numeric types"); + } + counts = NumericFacets.getCounts(searcher, docs, field, offset, limit, mincount, missing, sort); + } else { + PerSegmentSingleValuedFaceting ps = new PerSegmentSingleValuedFaceting(searcher, docs, field, offset,limit, mincount, missing, sort, prefix); + Executor executor = threads == 0 ? directExecutor : facetExecutor; + ps.setNumThreads(threads); + counts = ps.getFacetCounts(executor); + } } else { counts = getFieldCacheCounts(searcher, docs, field, offset,limit, mincount, missing, sort, prefix); } diff --git a/solr/core/src/java/org/apache/solr/request/UnInvertedField.java b/solr/core/src/java/org/apache/solr/request/UnInvertedField.java index 7407e790e43..0a106bb4aeb 100755 --- a/solr/core/src/java/org/apache/solr/request/UnInvertedField.java +++ b/solr/core/src/java/org/apache/solr/request/UnInvertedField.java @@ -483,13 +483,7 @@ public class UnInvertedField extends DocTermOrds { SortedDocValues si; for (String f : facet) { SchemaField facet_sf = searcher.getSchema().getField(f); - try { - si = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), f); - } - catch (IOException e) { - throw new RuntimeException("failed to open field cache for: " + f, e); - } - finfo[i] = new FieldFacetStats(f, si, sf, facet_sf, numTermsInField); + finfo[i] = new FieldFacetStats(searcher, f, sf, facet_sf); i++; } diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java index 4721e9e0800..c48ddf40fac 100644 --- a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java @@ -51,6 +51,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -103,7 +107,7 @@ public abstract class AbstractSpatialFieldType extend } @Override - public Field[] createFields(SchemaField field, Object val, float boost) { + public List createFields(SchemaField field, Object val, float boost) { String shapeStr = null; Shape shape = null; if (val instanceof Shape) { @@ -114,34 +118,22 @@ public abstract class AbstractSpatialFieldType extend } if (shape == null) { log.debug("Field {}: null shape for input: {}", field, val); - return null; + return Collections.emptyList(); } - Field[] indexableFields = null; + List result = new ArrayList(); if (field.indexed()) { T strategy = getStrategy(field.getName()); - indexableFields = strategy.createIndexableFields(shape); + result.addAll(Arrays.asList(strategy.createIndexableFields(shape))); } - StoredField storedField = null; if (field.stored()) { if (shapeStr == null) shapeStr = shapeToString(shape); - storedField = new StoredField(field.getName(), shapeStr); + result.add(new StoredField(field.getName(), shapeStr)); } - if (indexableFields == null) { - if (storedField == null) - return null; - return new Field[]{storedField}; - } else { - if (storedField == null) - return indexableFields; - Field[] result = new Field[indexableFields.length+1]; - System.arraycopy(indexableFields,0,result,0,indexableFields.length); - result[result.length-1] = storedField; - return result; - } + return result; } protected Shape parseShape(String shapeStr) { diff --git a/solr/core/src/java/org/apache/solr/schema/CurrencyField.java b/solr/core/src/java/org/apache/solr/schema/CurrencyField.java index 0575a3d9670..32b7ce7e5b9 100644 --- a/solr/core/src/java/org/apache/solr/schema/CurrencyField.java +++ b/solr/core/src/java/org/apache/solr/schema/CurrencyField.java @@ -46,9 +46,11 @@ import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.Currency; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -145,14 +147,14 @@ public class CurrencyField extends FieldType implements SchemaAware, ResourceLoa } @Override - public StorableField[] createFields(SchemaField field, Object externalVal, float boost) { + public List createFields(SchemaField field, Object externalVal, float boost) { CurrencyValue value = CurrencyValue.parse(externalVal.toString(), defaultCurrency); - StorableField[] f = new StorableField[field.stored() ? 3 : 2]; + List f = new ArrayList(); SchemaField amountField = getAmountField(field); - f[0] = amountField.createField(String.valueOf(value.getAmount()), amountField.indexed() && !amountField.omitNorms() ? boost : 1F); + f.add(amountField.createField(String.valueOf(value.getAmount()), amountField.indexed() && !amountField.omitNorms() ? boost : 1F)); SchemaField currencyField = getCurrencyField(field); - f[1] = currencyField.createField(value.getCurrencyCode(), currencyField.indexed() && !currencyField.omitNorms() ? boost : 1F); + f.add(currencyField.createField(value.getCurrencyCode(), currencyField.indexed() && !currencyField.omitNorms() ? boost : 1F)); if (field.stored()) { org.apache.lucene.document.FieldType customType = new org.apache.lucene.document.FieldType(); @@ -162,7 +164,7 @@ public class CurrencyField extends FieldType implements SchemaAware, ResourceLoa if (storedValue.indexOf(",") < 0) { storedValue += "," + defaultCurrency; } - f[2] = createField(field.getName(), storedValue, customType, 1F); + f.add(createField(field.getName(), storedValue, customType, 1F)); } return f; diff --git a/solr/core/src/java/org/apache/solr/schema/DateField.java b/solr/core/src/java/org/apache/solr/schema/DateField.java index ac934cf7d36..f047e1a0650 100644 --- a/solr/core/src/java/org/apache/solr/schema/DateField.java +++ b/solr/core/src/java/org/apache/solr/schema/DateField.java @@ -435,7 +435,7 @@ public class DateField extends PrimitiveFieldType { @Override public ValueSource getValueSource(SchemaField field, QParser parser) { field.checkFieldCacheSource(parser); - return new DateFieldSource(field.getName(), field.getType()); + return new DateFieldSource(field.getName(), field); } /** DateField specific range query */ @@ -453,11 +453,13 @@ public class DateField extends PrimitiveFieldType { class DateFieldSource extends FieldCacheSource { // NOTE: this is bad for serialization... but we currently need the fieldType for toInternal() + SchemaField sf; FieldType ft; - public DateFieldSource(String name, FieldType ft) { + public DateFieldSource(String name, SchemaField sf) { super(name); - this.ft = ft; + this.sf = sf; + this.ft = sf.getType(); } @Override @@ -474,6 +476,11 @@ class DateFieldSource extends FieldCacheSource { return ft.toInternal(readableValue); } + @Override + public boolean exists(int doc) { + return termsIndex.getOrd(doc) >= 0; + } + @Override public float floatVal(int doc) { return (float)intVal(doc); @@ -514,7 +521,7 @@ class DateFieldSource extends FieldCacheSource { } else { final BytesRef br = new BytesRef(); termsIndex.lookupOrd(ord, br); - return ft.toObject(null, br); + return ft.toObject(sf, br); } } diff --git a/solr/core/src/java/org/apache/solr/schema/FieldProperties.java b/solr/core/src/java/org/apache/solr/schema/FieldProperties.java index 137d86c2016..3a2b987b72c 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldProperties.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldProperties.java @@ -50,6 +50,7 @@ public abstract class FieldProperties { protected final static int OMIT_POSITIONS = 0x00002000; protected final static int STORE_OFFSETS = 0x00004000; + protected final static int DOC_VALUES = 0x00008000; static final String[] propertyNames = { "indexed", "tokenized", "stored", @@ -57,7 +58,7 @@ public abstract class FieldProperties { "termVectors", "termPositions", "termOffsets", "multiValued", "sortMissingFirst","sortMissingLast","required", "omitPositions", - "storeOffsetsWithPositions" + "storeOffsetsWithPositions", "docValues" }; static final Map propertyMap = new HashMap(); diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 9ba293c4a21..3bf13f17b51 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -17,14 +17,20 @@ package org.apache.solr.schema; +import java.io.IOException; +import java.io.Reader; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.document.Field; +import org.apache.lucene.index.FieldInfo.DocValuesType; import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.GeneralField; -import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.StorableField; import org.apache.lucene.index.Term; import org.apache.lucene.queries.function.ValueSource; @@ -45,11 +51,6 @@ import org.apache.solr.search.Sorting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.Reader; -import java.util.HashMap; -import java.util.Map; - /** * Base class for all field types used by an index schema. * @@ -120,14 +121,6 @@ public abstract class FieldType extends FieldProperties { } - protected String getArg(String n, Map args) { - String s = args.remove(n); - if (s == null) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Missing parameter '"+n+"' for FieldType=" + typeName +args); - } - return s; - } - // Handle additional arguments... void setArgs(IndexSchema schema, Map args) { // default to STORED, INDEXED, OMIT_TF_POSITIONS and MULTIVALUED depending on schema version @@ -169,11 +162,8 @@ public abstract class FieldType extends FieldProperties { initArgs.remove("positionIncrementGap"); } - final String postingsFormat = initArgs.get("postingsFormat"); - if (postingsFormat != null) { - this.postingsFormat = postingsFormat; - initArgs.remove("postingsFormat"); - } + this.postingsFormat = initArgs.remove("postingsFormat"); + this.docValuesFormat = initArgs.remove("docValuesFormat"); if (initArgs.size() > 0) { throw new RuntimeException("schema fieldtype " + typeName @@ -261,7 +251,7 @@ public abstract class FieldType extends FieldProperties { newType.setStoreTermVectors(field.storeTermVector()); newType.setStoreTermVectorOffsets(field.storeTermOffsets()); newType.setStoreTermVectorPositions(field.storeTermPositions()); - + return createField(field.getName(), val, newType, boost); } @@ -290,9 +280,15 @@ public abstract class FieldType extends FieldProperties { * @see #createField(SchemaField, Object, float) * @see #isPolyField() */ - public StorableField[] createFields(SchemaField field, Object value, float boost) { + public List createFields(SchemaField field, Object value, float boost) { StorableField f = createField( field, value, boost); - return f==null ? new StorableField[]{} : new StorableField[]{f}; + if (field.hasDocValues() && f.fieldType().docValueType() == null) { + // field types that support doc values should either override createField + // to return a field with doc values or extend createFields if this can't + // be done in a single field instance (see StrField for example) + throw new UnsupportedOperationException("This field type does not support doc values: " + this); + } + return f==null ? Collections.emptyList() : Collections.singletonList(f); } protected IndexOptions getIndexOptions(SchemaField field, String internalVal) { @@ -513,7 +509,13 @@ public abstract class FieldType extends FieldProperties { public Similarity getSimilarity() { return similarity; } - + + /** Return the numeric type of this field, or null if this field is not a + * numeric field. */ + public org.apache.lucene.document.FieldType.NumericType getNumericType() { + return null; + } + /** * Sets the Similarity used when scoring fields of this type * @lucene.internal @@ -530,7 +532,16 @@ public abstract class FieldType extends FieldProperties { public String getPostingsFormat() { return postingsFormat; } - + + /** + * The docvalues format used for this field type + */ + protected String docValuesFormat; + + public final String getDocValuesFormat() { + return docValuesFormat; + } + /** * calls back to TextResponseWriter to write the field value */ @@ -562,7 +573,6 @@ public abstract class FieldType extends FieldProperties { return new StrFieldSource(field.name); } - /** * Returns a Query instance for doing range searches on this field type. {@link org.apache.solr.search.SolrQueryParser} * currently passes part1 and part2 as null if they are '*' respectively. minInclusive and maxInclusive are both true @@ -615,7 +625,11 @@ public abstract class FieldType extends FieldProperties { * if invariants are violated by the SchemaField. *

*/ - public void checkSchemaField(final SchemaField field) throws SolrException { - // :NOOP: + public void checkSchemaField(final SchemaField field) { + // override if your field type supports doc values + if (field.hasDocValues()) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Field type " + this + " does not support doc values"); + } } + } diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonType.java b/solr/core/src/java/org/apache/solr/schema/LatLonType.java index d9fe7393677..04ecd4356db 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonType.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonType.java @@ -16,31 +16,43 @@ package org.apache.solr.schema; * limitations under the License. */ -import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.StorableField; -import org.apache.lucene.queries.function.FunctionValues; -import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.queries.function.valuesource.VectorValueSource; -import org.apache.lucene.search.*; -import com.spatial4j.core.io.ParseUtils; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.exception.InvalidShapeException; -import com.spatial4j.core.shape.Rectangle; -import org.apache.lucene.util.Bits; -import org.apache.solr.common.SolrException; -import org.apache.solr.response.TextResponseWriter; -import org.apache.solr.search.*; -import org.apache.solr.search.function.distance.HaversineConstFunction; - import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.StorableField; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.VectorValueSource; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ComplexExplanation; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.Bits; +import org.apache.solr.common.SolrException; +import org.apache.solr.response.TextResponseWriter; +import org.apache.solr.search.DelegatingCollector; +import org.apache.solr.search.ExtendedQueryBase; +import org.apache.solr.search.PostFilter; +import org.apache.solr.search.QParser; +import org.apache.solr.search.SpatialOptions; + +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.distance.DistanceUtils; +import com.spatial4j.core.exception.InvalidShapeException; +import com.spatial4j.core.io.ParseUtils; +import com.spatial4j.core.shape.Rectangle; + /** * Represents a Latitude/Longitude as a 2 dimensional point. Latitude is always specified first. @@ -57,10 +69,10 @@ public class LatLonType extends AbstractSubTypeFieldType implements SpatialQuery } @Override - public StorableField[] createFields(SchemaField field, Object value, float boost) { + public List createFields(SchemaField field, Object value, float boost) { String externalVal = value.toString(); //we could have tileDiff + 3 fields (two for the lat/lon, one for storage) - StorableField[] f = new StorableField[(field.indexed() ? 2 : 0) + (field.stored() ? 1 : 0)]; + List f = new ArrayList(3); if (field.indexed()) { int i = 0; double[] latLon; @@ -71,18 +83,18 @@ public class LatLonType extends AbstractSubTypeFieldType implements SpatialQuery } //latitude SchemaField lat = subField(field, i); - f[i] = lat.createField(String.valueOf(latLon[LAT]), lat.indexed() && !lat.omitNorms() ? boost : 1f); + f.add(lat.createField(String.valueOf(latLon[LAT]), lat.indexed() && !lat.omitNorms() ? boost : 1f)); i++; //longitude SchemaField lon = subField(field, i); - f[i] = lon.createField(String.valueOf(latLon[LON]), lon.indexed() && !lon.omitNorms() ? boost : 1f); + f.add(lon.createField(String.valueOf(latLon[LON]), lon.indexed() && !lon.omitNorms() ? boost : 1f)); } if (field.stored()) { FieldType customType = new FieldType(); customType.setStored(true); - f[f.length - 1] = createField(field.getName(), externalVal, customType, 1f); + f.add(createField(field.getName(), externalVal, customType, 1f)); } return f; } diff --git a/solr/core/src/java/org/apache/solr/schema/PointType.java b/solr/core/src/java/org/apache/solr/schema/PointType.java index 0697db7dfb9..d70c66194e4 100644 --- a/solr/core/src/java/org/apache/solr/schema/PointType.java +++ b/solr/core/src/java/org/apache/solr/schema/PointType.java @@ -69,7 +69,7 @@ public class PointType extends CoordinateFieldType implements SpatialQueryable { } @Override - public StorableField[] createFields(SchemaField field, Object value, float boost) { + public List createFields(SchemaField field, Object value, float boost) { String externalVal = value.toString(); String[] point = new String[0]; try { @@ -79,12 +79,12 @@ public class PointType extends CoordinateFieldType implements SpatialQueryable { } // TODO: this doesn't currently support polyFields as sub-field types - StorableField[] f = new StorableField[ (field.indexed() ? dimension : 0) + (field.stored() ? 1 : 0) ]; + List f = new ArrayList(dimension+1); if (field.indexed()) { for (int i=0; i createFields(Object val, float boost) { return type.createFields(this,val,boost); } @@ -148,9 +149,9 @@ public final class SchemaField extends FieldProperties { * @see FieldType#getSortField */ public void checkSortability() throws SolrException { - if (! indexed() ) { + if (! (indexed() || hasDocValues()) ) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "can not sort on unindexed field: " + "can not sort on a field which is neither indexed nor has doc values: " + getName()); } if ( multiValued() ) { @@ -169,9 +170,9 @@ public final class SchemaField extends FieldProperties { * @see FieldType#getValueSource */ public void checkFieldCacheSource(QParser parser) throws SolrException { - if (! indexed() ) { + if (! (indexed() || hasDocValues()) ) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "can not use FieldCache on unindexed field: " + "can not use FieldCache on a field which is neither indexed nor has doc values: " + getName()); } if ( multiValued() ) { diff --git a/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java b/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java index 10c1140e4a1..65436f4bd6f 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java @@ -131,6 +131,11 @@ class SortableDoubleFieldSource extends FieldCacheSource { return NumberUtils.double2sortableStr(readableValue); } + @Override + public boolean exists(int doc) { + return termsIndex.getOrd(doc) >= 0; + } + @Override public float floatVal(int doc) { return (float)doubleVal(doc); @@ -164,13 +169,7 @@ class SortableDoubleFieldSource extends FieldCacheSource { @Override public Object objectVal(int doc) { - int ord=termsIndex.getOrd(doc); - if (ord==-1) { - return null; - } else { - termsIndex.lookupOrd(ord, spare); - return NumberUtils.SortableStr2double(spare); - } + return exists(doc) ? doubleVal(doc) : null; } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java b/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java index 9635e6f1bd5..69db7616b2f 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java @@ -135,6 +135,11 @@ class SortableFloatFieldSource extends FieldCacheSource { return NumberUtils.float2sortableStr(readableValue); } + @Override + public boolean exists(int doc) { + return termsIndex.getOrd(doc) >= 0; + } + @Override public float floatVal(int doc) { int ord=termsIndex.getOrd(doc); @@ -173,13 +178,7 @@ class SortableFloatFieldSource extends FieldCacheSource { @Override public Object objectVal(int doc) { - int ord=termsIndex.getOrd(doc); - if (ord==-1) { - return null; - } else { - termsIndex.lookupOrd(ord, spare); - return NumberUtils.SortableStr2float(spare); - } + return exists(doc) ? floatVal(doc) : null; } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/SortableIntField.java b/solr/core/src/java/org/apache/solr/schema/SortableIntField.java index 1a850aa0c08..cbcb913ea84 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableIntField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableIntField.java @@ -142,6 +142,11 @@ class SortableIntFieldSource extends FieldCacheSource { return (float)intVal(doc); } + @Override + public boolean exists(int doc) { + return termsIndex.getOrd(doc) >= 0; + } + @Override public int intVal(int doc) { int ord=termsIndex.getOrd(doc); @@ -175,13 +180,7 @@ class SortableIntFieldSource extends FieldCacheSource { @Override public Object objectVal(int doc) { - int ord=termsIndex.getOrd(doc); - if (ord==-1) { - return null; - } else { - termsIndex.lookupOrd(ord, spare); - return NumberUtils.SortableStr2int(spare); - } + return exists(doc) ? intVal(doc) : null; } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/SortableLongField.java b/solr/core/src/java/org/apache/solr/schema/SortableLongField.java index 8ce95ce5dcc..0e61eef6f91 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableLongField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableLongField.java @@ -135,6 +135,11 @@ class SortableLongFieldSource extends FieldCacheSource { return NumberUtils.long2sortableStr(readableValue); } + @Override + public boolean exists(int doc) { + return termsIndex.getOrd(doc) >= 0; + } + @Override public float floatVal(int doc) { return (float)longVal(doc); @@ -168,13 +173,7 @@ class SortableLongFieldSource extends FieldCacheSource { @Override public Object objectVal(int doc) { - int ord=termsIndex.getOrd(doc); - if (ord==-1) { - return null; - } else { - termsIndex.lookupOrd(ord, spare); - return NumberUtils.SortableStr2long(spare); - } + return exists(doc) ? longVal(doc) : null; } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/StrField.java b/solr/core/src/java/org/apache/solr/schema/StrField.java index a6e81650e77..4f370cd3f3b 100644 --- a/solr/core/src/java/org/apache/solr/schema/StrField.java +++ b/solr/core/src/java/org/apache/solr/schema/StrField.java @@ -17,20 +17,43 @@ package org.apache.solr.schema; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.index.StorableField; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.SortField; -import org.apache.lucene.index.GeneralField; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.StorableField; import org.apache.lucene.util.BytesRef; import org.apache.solr.response.TextResponseWriter; import org.apache.solr.search.QParser; -import java.io.IOException; -/** - * - */ public class StrField extends PrimitiveFieldType { + + @Override + protected void init(IndexSchema schema, Map args) { + super.init(schema, args); + } + + @Override + public List createFields(SchemaField field, Object value, + float boost) { + if (field.hasDocValues()) { + List fields = new ArrayList(); + fields.add(createField(field, value, boost)); + final BytesRef bytes = new BytesRef(value.toString()); + final Field docValuesField = new SortedDocValuesField(field.getName(), bytes); + fields.add(docValuesField); + return fields; + } else { + return Collections.singletonList(createField(field, value, boost)); + } + } + @Override public SortField getSortField(SchemaField field,boolean reverse) { return getStringSort(field,reverse); @@ -51,6 +74,14 @@ public class StrField extends PrimitiveFieldType { public Object toObject(SchemaField sf, BytesRef term) { return term.utf8ToString(); } + + @Override + public void checkSchemaField(SchemaField field) { + // change me when multi-valued doc values are supported + if (field.hasDocValues() && !(field.isRequired() || field.getDefaultValue() != null)) { + throw new IllegalStateException("Field " + this + " has doc values enabled, but has no default value and is not required"); + } + } } diff --git a/solr/core/src/java/org/apache/solr/schema/TrieDateField.java b/solr/core/src/java/org/apache/solr/schema/TrieDateField.java index fddcdbba897..3ac0c78e5f0 100755 --- a/solr/core/src/java/org/apache/solr/schema/TrieDateField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieDateField.java @@ -17,11 +17,13 @@ package org.apache.solr.schema; +import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.solr.search.QParser; +import org.apache.solr.common.SolrException; import org.apache.solr.response.TextResponseWriter; -import org.apache.lucene.index.GeneralField; -import org.apache.lucene.index.IndexableField; +import org.apache.lucene.document.FieldType.NumericType; +import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.StorableField; import org.apache.lucene.search.SortField; import org.apache.lucene.search.Query; @@ -29,6 +31,7 @@ import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRef; +import java.util.List; import java.util.Map; import java.util.Date; import java.io.IOException; @@ -73,6 +76,10 @@ public class TrieDateField extends DateField { return wrappedField.getPrecisionStep(); } + @Override + public NumericType getNumericType() { + return wrappedField.getNumericType(); + } @Override public void write(TextResponseWriter writer, String name, StorableField f) throws IOException { @@ -129,6 +136,11 @@ public class TrieDateField extends DateField { return wrappedField.createField(field, value, boost); } + @Override + public List createFields(SchemaField field, Object value, float boost) { + return wrappedField.createFields(field, value, boost); + } + @Override public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) { return wrappedField.getRangeQuery(parser, field, min, max, minInclusive, maxInclusive); @@ -141,4 +153,10 @@ public class TrieDateField extends DateField { max == null ? null : max.getTime(), minInclusive, maxInclusive); } + + @Override + public void checkSchemaField(SchemaField field) { + wrappedField.checkSchemaField(field); + } + } diff --git a/solr/core/src/java/org/apache/solr/schema/TrieField.java b/solr/core/src/java/org/apache/solr/schema/TrieField.java index afe4da286b0..59c20236d1f 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieField.java @@ -17,7 +17,10 @@ package org.apache.solr.schema; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.Date; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -29,13 +32,17 @@ import org.apache.lucene.document.FieldType.NumericType; import org.apache.lucene.document.FloatField; import org.apache.lucene.document.IntField; import org.apache.lucene.document.LongField; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.StorableField; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.DoubleFieldSource; import org.apache.lucene.queries.function.valuesource.FloatFieldSource; import org.apache.lucene.queries.function.valuesource.IntFieldSource; import org.apache.lucene.queries.function.valuesource.LongFieldSource; -import org.apache.lucene.search.*; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.NumericUtils; @@ -101,8 +108,7 @@ public class TrieField extends PrimitiveFieldType { "Invalid type specified in schema.xml for field: " + args.get("name"), e); } } - - + CharFilterFactory[] filterFactories = new CharFilterFactory[0]; TokenFilterFactory[] tokenFilterFactories = new TokenFilterFactory[0]; analyzer = new TokenizerChain(filterFactories, new TrieTokenizerFactory(type, precisionStep), tokenFilterFactories); @@ -236,6 +242,23 @@ public class TrieField extends PrimitiveFieldType { return type; } + @Override + public NumericType getNumericType() { + switch (type) { + case INTEGER: + return NumericType.INT; + case LONG: + case DATE: + return NumericType.LONG; + case FLOAT: + return NumericType.FLOAT; + case DOUBLE: + return NumericType.DOUBLE; + default: + throw new AssertionError(); + } + } + @Override public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) { int ps = precisionStep; @@ -473,8 +496,9 @@ public class TrieField extends PrimitiveFieldType { public StorableField createField(SchemaField field, Object value, float boost) { boolean indexed = field.indexed(); boolean stored = field.stored(); + boolean docValues = field.hasDocValues(); - if (!indexed && !stored) { + if (!indexed && !stored && !docValues) { if (log.isTraceEnabled()) log.trace("Ignoring unindexed/unstored field: " + field); return null; @@ -549,6 +573,28 @@ public class TrieField extends PrimitiveFieldType { return f; } + @Override + public List createFields(SchemaField sf, Object value, float boost) { + if (sf.hasDocValues()) { + List fields = new ArrayList(); + final StorableField field = createField(sf, value, boost); + fields.add(field); + final long bits; + if (field.numericValue() instanceof Integer || field.numericValue() instanceof Long) { + bits = field.numericValue().longValue(); + } else if (field.numericValue() instanceof Float) { + bits = Float.floatToIntBits(field.numericValue().floatValue()); + } else { + assert field.numericValue() instanceof Double; + bits = Double.doubleToLongBits(field.numericValue().doubleValue()); + } + fields.add(new NumericDocValuesField(sf.getName(), bits)); + return fields; + } else { + return Collections.singletonList(createField(sf, value, boost)); + } + } + public enum TrieTypes { INTEGER, LONG, @@ -586,6 +632,13 @@ public class TrieField extends PrimitiveFieldType { } return null; } + + @Override + public void checkSchemaField(final SchemaField field) { + if (field.hasDocValues() && !(field.isRequired() || field.getDefaultValue() != null)) { + throw new IllegalStateException("Field " + this + " has doc values enabled, but has no default value and is not required"); + } + } } class TrieDateFieldSource extends LongFieldSource { @@ -605,14 +658,20 @@ class TrieDateFieldSource extends LongFieldSource { } @Override - public Object longToObject(long val) { + public Date longToObject(long val) { return new Date(val); } + @Override + public String longToString(long val) { + return TrieField.dateField.toExternal(longToObject(val)); + } + @Override public long externalToLong(String extVal) { return TrieField.dateField.parseMath(null, extVal).getTime(); } + } diff --git a/solr/core/src/java/org/apache/solr/schema/UUIDField.java b/solr/core/src/java/org/apache/solr/schema/UUIDField.java index df45a705944..33c95b72c51 100644 --- a/solr/core/src/java/org/apache/solr/schema/UUIDField.java +++ b/solr/core/src/java/org/apache/solr/schema/UUIDField.java @@ -22,8 +22,6 @@ import java.util.Locale; import java.util.Map; import java.util.UUID; -import org.apache.lucene.index.GeneralField; -import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.StorableField; import org.apache.lucene.search.SortField; import org.apache.solr.common.SolrException; diff --git a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java index 1729951020b..6ce5c9ede38 100644 --- a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java +++ b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java @@ -23,10 +23,7 @@ import java.util.List; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.StorableField; -import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; @@ -58,33 +55,19 @@ public class DocumentBuilder { // we don't check for a null val ourselves because a solr.FieldType // might actually want to map it to something. If createField() // returns null, then we don't store the field. - if (sfield.isPolyField()) { - StorableField[] fields = sfield.createFields(val, boost); - if (fields.length > 0) { - if (!sfield.multiValued()) { - String oldValue = map.put(sfield.getName(), val); - if (oldValue != null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ERROR: multiple values encountered for non multiValued field " + sfield.getName() - + ": first='" + oldValue + "' second='" + val + "'"); - } - } - // Add each field - for (StorableField field : fields) { - doc.add((Field) field); + List fields = sfield.createFields(val, boost); + if (!fields.isEmpty()) { + if (!sfield.multiValued()) { + String oldValue = map.put(sfield.getName(), val); + if (oldValue != null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ERROR: multiple values encountered for non multiValued field " + sfield.getName() + + ": first='" + oldValue + "' second='" + val + "'"); } } - } else { - StorableField field = sfield.createField(val, boost); - if (field != null) { - if (!sfield.multiValued()) { - String oldValue = map.put(sfield.getName(), val); - if (oldValue != null) { - throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"ERROR: multiple values encountered for non multiValued field " + sfield.getName() - + ": first='" + oldValue + "' second='" + val + "'"); - } - } + // Add each field + for (StorableField field : fields) { + doc.add((Field) field); } - doc.add((Field) field); } } @@ -192,14 +175,8 @@ public class DocumentBuilder { private static void addField(Document doc, SchemaField field, Object val, float boost) { - if (field.isPolyField()) { - StorableField[] farr = field.getType().createFields(field, val, boost); - for (StorableField f : farr) { - if (f != null) doc.add((Field) f); // null fields are not added - } - } else { - StorableField f = field.createField(val, boost); - if (f != null) doc.add((Field) f); // null fields are not added + for (StorableField f : field.getType().createFields(field, val, boost)) { + if (f != null) doc.add((Field) f); // null fields are not added } } diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-docValues-not-required-no-default.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-docValues-not-required-no-default.xml new file mode 100644 index 00000000000..deadd9ac68b --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-docValues-not-required-no-default.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + id + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-docValues.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-docValues.xml new file mode 100644 index 00000000000..5f4d69a31a7 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-docValues.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml new file mode 100644 index 00000000000..63d87997402 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml @@ -0,0 +1,74 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema.xml b/solr/core/src/test-files/solr/collection1/conf/schema.xml index 417d7dbf5ca..cfa31d1f1d2 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema.xml @@ -567,7 +567,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml b/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml index e28cec73722..15074809892 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml @@ -20,18 +20,29 @@ - - + + + + + + - + + + + + + - + + + string_f string_f diff --git a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java index a49fbf98397..049723fb739 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java +++ b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java @@ -20,6 +20,7 @@ package org.apache.solr.core; import java.util.Map; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.schema.SchemaField; @@ -47,7 +48,21 @@ public class TestCodecSupport extends SolrTestCaseJ4 { assertEquals("Lucene41", format.getPostingsFormatForField(schemaField.getName()).getName()); } - public void testDynamicFields() { + public void testDocValuesFormats() { + Codec codec = h.getCore().getCodec(); + Map fields = h.getCore().getSchema().getFields(); + SchemaField schemaField = fields.get("string_disk_f"); + PerFieldDocValuesFormat format = (PerFieldDocValuesFormat) codec.docValuesFormat(); + assertEquals("Disk", format.getDocValuesFormatForField(schemaField.getName()).getName()); + schemaField = fields.get("string_memory_f"); + assertEquals("Lucene42", + format.getDocValuesFormatForField(schemaField.getName()).getName()); + schemaField = fields.get("string_f"); + assertEquals("Lucene42", + format.getDocValuesFormatForField(schemaField.getName()).getName()); + } + + public void testDynamicFieldsPostingsFormats() { Codec codec = h.getCore().getCodec(); PerFieldPostingsFormat format = (PerFieldPostingsFormat) codec.postingsFormat(); @@ -59,6 +74,16 @@ public class TestCodecSupport extends SolrTestCaseJ4 { assertEquals("Lucene41", format.getPostingsFormatForField("bar_standard").getName()); } + public void testDynamicFieldsDocValuesFormats() { + Codec codec = h.getCore().getCodec(); + PerFieldDocValuesFormat format = (PerFieldDocValuesFormat) codec.docValuesFormat(); + + assertEquals("Disk", format.getDocValuesFormatForField("foo_disk").getName()); + assertEquals("Disk", format.getDocValuesFormatForField("bar_disk").getName()); + assertEquals("Lucene42", format.getDocValuesFormatForField("foo_memory").getName()); + assertEquals("Lucene42", format.getDocValuesFormatForField("bar_memory").getName()); + } + public void testUnknownField() { Codec codec = h.getCore().getCodec(); PerFieldPostingsFormat format = (PerFieldPostingsFormat) codec.postingsFormat(); diff --git a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java index 78cf362bdc3..0fa83fea53e 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java @@ -75,6 +75,7 @@ public class StatsComponentTest extends AbstractSolrTestCase { public void doTestFieldStatisticsResult(String f) throws Exception { assertU(adoc("id", "1", f, "-10")); assertU(adoc("id", "2", f, "-20")); + assertU(commit()); assertU(adoc("id", "3", f, "-30")); assertU(adoc("id", "4", f, "-40")); assertU(commit()); @@ -205,6 +206,7 @@ public class StatsComponentTest extends AbstractSolrTestCase { public void doTestFieldStatisticsMissingResult(String f) throws Exception { assertU(adoc("id", "1", f, "-10")); assertU(adoc("id", "2", f, "-20")); + assertU(commit()); assertU(adoc("id", "3")); assertU(adoc("id", "4", f, "-40")); assertU(commit()); @@ -224,6 +226,7 @@ public class StatsComponentTest extends AbstractSolrTestCase { public void doTestFacetStatisticsResult(String f) throws Exception { assertU(adoc("id", "1", f, "10", "active_s", "true", "other_s", "foo")); assertU(adoc("id", "2", f, "20", "active_s", "true", "other_s", "bar")); + assertU(commit()); assertU(adoc("id", "3", f, "30", "active_s", "false", "other_s", "foo")); assertU(adoc("id", "4", f, "40", "active_s", "false", "other_s", "foo")); assertU(commit()); @@ -257,6 +260,7 @@ public class StatsComponentTest extends AbstractSolrTestCase { public void doTestFacetStatisticsMissingResult(String f) throws Exception { assertU(adoc("id", "1", f, "10", "active_s", "true")); assertU(adoc("id", "2", f, "20", "active_s", "true")); + assertU(commit()); assertU(adoc("id", "3", "active_s", "false")); assertU(adoc("id", "4", f, "40", "active_s", "false")); assertU(commit()); @@ -288,6 +292,7 @@ public class StatsComponentTest extends AbstractSolrTestCase { SolrCore core = h.getCore(); assertU(adoc("id", "1")); assertU(adoc("id", "2")); + assertU(commit()); assertU(adoc("id", "3")); assertU(adoc("id", "4")); assertU(commit()); @@ -307,6 +312,7 @@ public class StatsComponentTest extends AbstractSolrTestCase { SolrCore core = h.getCore(); assertU(adoc("id", "1")); assertU(adoc("id", "2")); + assertU(commit()); assertU(adoc("id", "3")); assertU(adoc("id", "4")); assertU(commit()); @@ -328,6 +334,7 @@ public class StatsComponentTest extends AbstractSolrTestCase { assertU(adoc("id", "1")); assertU(adoc("id", "2")); + assertU(commit()); assertU(adoc("id", "3")); assertU(commit()); @@ -347,6 +354,7 @@ public class StatsComponentTest extends AbstractSolrTestCase { SchemaField foo_ss = core.getSchema().getField("foo_ss"); assertU(adoc("id", "1", "active_i", "1", "foo_ss", "aa" )); + assertU(commit()); assertU(adoc("id", "2", "active_i", "1", "foo_ss", "bb" )); assertU(adoc("id", "3", "active_i", "5", "foo_ss", "aa" )); assertU(commit()); diff --git a/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java b/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java index 6c9e6fbf18a..746fd48f1b1 100644 --- a/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java +++ b/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java @@ -19,8 +19,6 @@ package org.apache.solr.schema; import org.apache.solr.core.AbstractBadConfigTestBase; -import java.util.regex.Pattern; - public class BadIndexSchemaTest extends AbstractBadConfigTestBase { private void doTest(final String schema, final String errString) @@ -83,5 +81,12 @@ public class BadIndexSchemaTest extends AbstractBadConfigTestBase { doTest("bad-schema-codec-global-vs-ft-mismatch.xml", "codec does not support"); } + public void testDocValuesNotRequiredNoDefault() throws Exception { + doTest("bad-schema-docValues-not-required-no-default.xml", "has no default value and is not required"); + } + + public void testDocValuesUnsupported() throws Exception { + doTest("bad-schema-unsupported-docValues.xml", "does not support doc values"); + } } diff --git a/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTest.java b/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTest.java index 04f363730ba..4c3af76a9cc 100644 --- a/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTest.java @@ -23,6 +23,7 @@ import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; +import java.util.List; import java.util.Random; import java.util.Set; @@ -71,18 +72,18 @@ public class CurrencyFieldTest extends SolrTestCaseJ4 { FieldType tmp = amount.getType(); assertTrue(tmp instanceof CurrencyField); String currencyValue = "1.50,EUR"; - StorableField[] fields = amount.createFields(currencyValue, 2); - assertEquals(fields.length, 3); + List fields = amount.createFields(currencyValue, 2); + assertEquals(fields.size(), 3); // First field is currency code, second is value, third is stored. for (int i = 0; i < 3; i++) { - boolean hasValue = fields[i].readerValue() != null - || fields[i].numericValue() != null - || fields[i].stringValue() != null; - assertTrue("Doesn't have a value: " + fields[i], hasValue); + boolean hasValue = fields.get(i).readerValue() != null + || fields.get(i).numericValue() != null + || fields.get(i).stringValue() != null; + assertTrue("Doesn't have a value: " + fields.get(i), hasValue); } - assertEquals(schema.getFieldTypeByName("string").toExternal(fields[2]), "1.50,EUR"); + assertEquals(schema.getFieldTypeByName("string").toExternal(fields.get(2)), "1.50,EUR"); // A few tests on the provider directly ExchangeRateProvider p = ((CurrencyField) tmp).getProvider(); diff --git a/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java new file mode 100644 index 00000000000..374abf903b5 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java @@ -0,0 +1,230 @@ +package org.apache.solr.schema; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.FieldInfo.DocValuesType; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.core.SolrCore; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.RefCounted; +import org.junit.BeforeClass; + +public class DocValuesTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeTests() throws Exception { + initCore("solrconfig-basic.xml", "schema-docValues.xml"); + } + + public void setUp() throws Exception { + super.setUp(); + assertU(delQ("*:*")); + } + + public void testDocValues() throws IOException { + assertU(adoc("id", "1")); + commit(); + SolrCore core = h.getCoreInc(); + try { + final RefCounted searcherRef = core.openNewSearcher(true, true); + final SolrIndexSearcher searcher = searcherRef.get(); + try { + final AtomicReader reader = searcher.getAtomicReader(); + assertEquals(1, reader.numDocs()); + final FieldInfos infos = reader.getFieldInfos(); + assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("floatdv").getDocValuesType()); + assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("intdv").getDocValuesType()); + assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("doubledv").getDocValuesType()); + assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("longdv").getDocValuesType()); + assertEquals(DocValuesType.SORTED, infos.fieldInfo("stringdv").getDocValuesType()); + + assertEquals((long) Float.floatToIntBits(1), reader.getNumericDocValues("floatdv").get(0)); + assertEquals(2L, reader.getNumericDocValues("intdv").get(0)); + assertEquals(Double.doubleToLongBits(3), reader.getNumericDocValues("doubledv").get(0)); + assertEquals(4L, reader.getNumericDocValues("longdv").get(0)); + + final IndexSchema schema = core.getSchema(); + final SchemaField floatDv = schema.getField("floatdv"); + final SchemaField intDv = schema.getField("intdv"); + final SchemaField doubleDv = schema.getField("doubledv"); + final SchemaField longDv = schema.getField("longdv"); + + FunctionValues values = floatDv.getType().getValueSource(floatDv, null).getValues(null, searcher.getAtomicReader().leaves().get(0)); + assertEquals(1f, values.floatVal(0), 0f); + assertEquals(1f, values.objectVal(0)); + values = intDv.getType().getValueSource(intDv, null).getValues(null, searcher.getAtomicReader().leaves().get(0)); + assertEquals(2, values.intVal(0)); + assertEquals(2, values.objectVal(0)); + values = doubleDv.getType().getValueSource(doubleDv, null).getValues(null, searcher.getAtomicReader().leaves().get(0)); + assertEquals(3d, values.doubleVal(0), 0d); + assertEquals(3d, values.objectVal(0)); + values = longDv.getType().getValueSource(longDv, null).getValues(null, searcher.getAtomicReader().leaves().get(0)); + assertEquals(4L, values.longVal(0)); + assertEquals(4L, values.objectVal(0)); + } finally { + searcherRef.decref(); + } + } finally { + core.close(); + } + } + + public void testDocValuesSorting() { + assertU(adoc("id", "1", "floatdv", "2", "intdv", "3", "doubledv", "4", "longdv", "5", "datedv", "1995-12-31T23:59:59.999Z", "stringdv", "b")); + assertU(adoc("id", "2", "floatdv", "5", "intdv", "4", "doubledv", "3", "longdv", "2", "datedv", "1997-12-31T23:59:59.999Z", "stringdv", "a")); + assertU(adoc("id", "3", "floatdv", "3", "intdv", "1", "doubledv", "2", "longdv", "1", "datedv", "1996-12-31T23:59:59.999Z", "stringdv", "c")); + assertU(adoc("id", "4")); + assertU(commit()); + assertQ(req("q", "*:*", "sort", "floatdv desc", "rows", "1", "fl", "id"), + "//str[@name='id'][.='2']"); + assertQ(req("q", "*:*", "sort", "intdv desc", "rows", "1", "fl", "id"), + "//str[@name='id'][.='2']"); + assertQ(req("q", "*:*", "sort", "doubledv desc", "rows", "1", "fl", "id"), + "//str[@name='id'][.='1']"); + assertQ(req("q", "*:*", "sort", "longdv desc", "rows", "1", "fl", "id"), + "//str[@name='id'][.='1']"); + assertQ(req("q", "*:*", "sort", "datedv desc", "rows", "1", "fl", "id"), + "//str[@name='id'][.='2']"); + assertQ(req("q", "*:*", "sort", "stringdv desc", "rows", "1", "fl", "id"), + "//str[@name='id'][.='4']"); + assertQ(req("q", "*:*", "sort", "floatdv asc", "rows", "1", "fl", "id"), + "//str[@name='id'][.='4']"); + assertQ(req("q", "*:*", "sort", "intdv asc", "rows", "1", "fl", "id"), + "//str[@name='id'][.='3']"); + assertQ(req("q", "*:*", "sort", "doubledv asc", "rows", "1", "fl", "id"), + "//str[@name='id'][.='3']"); + assertQ(req("q", "*:*", "sort", "longdv asc", "rows", "1", "fl", "id"), + "//str[@name='id'][.='3']"); + assertQ(req("q", "*:*", "sort", "datedv asc", "rows", "1", "fl", "id"), + "//str[@name='id'][.='1']"); + assertQ(req("q", "*:*", "sort", "stringdv asc", "rows", "1", "fl", "id"), + "//str[@name='id'][.='2']"); + } + + public void testDocValuesFaceting() { + for (int i = 0; i < 50; ++i) { + assertU(adoc("id", "" + i)); + } + for (int i = 0; i < 50; ++i) { + if (rarely()) { + commit(); // to have several segments + } + assertU(adoc("id", "1000" + i, "floatdv", "" + i, "intdv", "" + i, "doubledv", "" + i, "longdv", "" + i, "datedv", (1900+i) + "-12-31T23:59:59.999Z", "stringdv", "abc" + i)); + } + assertU(commit()); + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "longdv", "facet.sort", "count", "facet.limit", "1"), + "//lst[@name='longdv']/int[@name='4'][.='51']"); + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "longdv", "facet.sort", "count", "facet.offset", "1", "facet.limit", "1"), + "//lst[@name='longdv']/int[@name='0'][.='1']"); + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "longdv", "facet.sort", "index", "facet.offset", "33", "facet.limit", "1", "facet.mincount", "1"), + "//lst[@name='longdv']/int[@name='33'][.='1']"); + + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "floatdv", "facet.sort", "count", "facet.limit", "1"), + "//lst[@name='floatdv']/int[@name='1.0'][.='51']"); + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "floatdv", "facet.sort", "count", "facet.offset", "1", "facet.limit", "-1", "facet.mincount", "1"), + "//lst[@name='floatdv']/int[@name='0.0'][.='1']"); + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "floatdv", "facet.sort", "index", "facet.offset", "33", "facet.limit", "1", "facet.mincount", "1"), + "//lst[@name='floatdv']/int[@name='33.0'][.='1']"); + + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "doubledv", "facet.sort", "count", "facet.limit", "1"), + "//lst[@name='doubledv']/int[@name='3.0'][.='51']"); + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "doubledv", "facet.sort", "count", "facet.offset", "1", "facet.limit", "-1", "facet.mincount", "1"), + "//lst[@name='doubledv']/int[@name='0.0'][.='1']"); + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "doubledv", "facet.sort", "index", "facet.offset", "33", "facet.limit", "1", "facet.mincount", "1"), + "//lst[@name='doubledv']/int[@name='33.0'][.='1']"); + + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "intdv", "facet.sort", "count", "facet.limit", "1"), + "//lst[@name='intdv']/int[@name='2'][.='51']"); + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "intdv", "facet.sort", "count", "facet.offset", "1", "facet.limit", "-1", "facet.mincount", "1"), + "//lst[@name='intdv']/int[@name='0'][.='1']"); + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "intdv", "facet.sort", "index", "facet.offset", "33", "facet.limit", "1", "facet.mincount", "1"), + "//lst[@name='intdv']/int[@name='33'][.='1']"); + + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "datedv", "facet.sort", "count", "facet.limit", "1"), + "//lst[@name='datedv']/int[@name='1995-12-31T23:59:59.999Z'][.='50']"); + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "datedv", "facet.sort", "count", "facet.offset", "1", "facet.limit", "-1", "facet.mincount", "1"), + "//lst[@name='datedv']/int[@name='1900-12-31T23:59:59.999Z'][.='1']"); + assertQ(req("q", "*:*", "facet", "true", "rows", "0", "facet.field", "datedv", "facet.sort", "index", "facet.offset", "33", "facet.limit", "1", "facet.mincount", "1"), + "//lst[@name='datedv']/int[@name='1933-12-31T23:59:59.999Z'][.='1']"); + } + + public void testDocValuesStats() { + for (int i = 0; i < 50; ++i) { + assertU(adoc("id", "1000" + i, "floatdv", "" + i%2, "intdv", "" + i%3, "doubledv", "" + i%4, "longdv", "" + i%5, "datedv", (1900+i%6) + "-12-31T23:59:59.999Z", "stringdv", "abc" + i%7)); + if (rarely()) { + commit(); // to have several segments + } + } + assertU(commit()); + + assertQ(req("q", "*:*", "stats", "true", "rows", "0", "stats.field", "stringdv"), + "//str[@name='min'][.='abc0']", + "//str[@name='max'][.='abc6']", + "//long[@name='count'][.='50']"); + + assertQ(req("q", "*:*", "stats", "true", "rows", "0", "stats.field", "floatdv"), + "//double[@name='min'][.='0.0']", + "//double[@name='max'][.='1.0']", + "//long[@name='count'][.='50']", + "//double[@name='sum'][.='25.0']", + "//double[@name='mean'][.='0.5']"); + + assertQ(req("q", "*:*", "stats", "true", "rows", "0", "stats.field", "intdv"), + "//double[@name='min'][.='0.0']", + "//double[@name='max'][.='2.0']", + "//long[@name='count'][.='50']", + "//double[@name='sum'][.='49.0']"); + + assertQ(req("q", "*:*", "stats", "true", "rows", "0", "stats.field", "doubledv"), + "//double[@name='min'][.='0.0']", + "//double[@name='max'][.='3.0']", + "//long[@name='count'][.='50']", + "//double[@name='sum'][.='73.0']"); + + assertQ(req("q", "*:*", "stats", "true", "rows", "0", "stats.field", "longdv"), + "//double[@name='min'][.='0.0']", + "//double[@name='max'][.='4.0']", + "//long[@name='count'][.='50']", + "//double[@name='sum'][.='100.0']", + "//double[@name='mean'][.='2.0']"); + + assertQ(req("q", "*:*", "stats", "true", "rows", "0", "stats.field", "datedv"), + "//date[@name='min'][.='1900-12-31T23:59:59.999Z']", + "//date[@name='max'][.='1905-12-31T23:59:59.999Z']", + "//long[@name='count'][.='50']"); + + assertQ(req("q", "*:*", "stats", "true", "rows", "0", "stats.field", "floatdv", "stats.facet", "intdv"), + "//lst[@name='intdv']/lst[@name='0']/long[@name='count'][.='17']", + "//lst[@name='intdv']/lst[@name='1']/long[@name='count'][.='17']", + "//lst[@name='intdv']/lst[@name='2']/long[@name='count'][.='16']"); + + assertQ(req("q", "*:*", "stats", "true", "rows", "0", "stats.field", "floatdv", "stats.facet", "datedv"), + "//lst[@name='datedv']/lst[@name='1900-12-31T23:59:59.999Z']/long[@name='count'][.='9']", + "//lst[@name='datedv']/lst[@name='1901-12-31T23:59:59.999Z']/long[@name='count'][.='9']", + "//lst[@name='datedv']/lst[@name='1902-12-31T23:59:59.999Z']/long[@name='count'][.='8']", + "//lst[@name='datedv']/lst[@name='1903-12-31T23:59:59.999Z']/long[@name='count'][.='8']", + "//lst[@name='datedv']/lst[@name='1904-12-31T23:59:59.999Z']/long[@name='count'][.='8']", + "//lst[@name='datedv']/lst[@name='1905-12-31T23:59:59.999Z']/long[@name='count'][.='8']"); + } + +} diff --git a/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java b/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java index 87973e36293..d92487a71f8 100644 --- a/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java @@ -16,8 +16,9 @@ package org.apache.solr.schema; * limitations under the License. */ +import java.util.List; + import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.StorableField; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -84,14 +85,14 @@ public class PolyFieldTest extends SolrTestCaseJ4 { assertEquals(pt.getDimension(), 2); double[] xy = new double[]{35.0, -79.34}; String point = xy[0] + "," + xy[1]; - StorableField[] fields = home.createFields(point, 2); - assertEquals(fields.length, 3);//should be 3, we have a stored field + List fields = home.createFields(point, 2); + assertEquals(fields.size(), 3);//should be 3, we have a stored field //first two fields contain the values, third is just stored and contains the original for (int i = 0; i < 3; i++) { - boolean hasValue = fields[i].binaryValue() != null - || fields[i].stringValue() != null - || fields[i].numericValue() != null; - assertTrue("Doesn't have a value: " + fields[i], hasValue); + boolean hasValue = fields.get(i).binaryValue() != null + || fields.get(i).stringValue() != null + || fields.get(i).numericValue() != null; + assertTrue("Doesn't have a value: " + fields.get(i), hasValue); } /*assertTrue("first field " + fields[0].tokenStreamValue() + " is not 35.0", pt.getSubType().toExternal(fields[0]).equals(String.valueOf(xy[0]))); assertTrue("second field is not -79.34", pt.getSubType().toExternal(fields[1]).equals(String.valueOf(xy[1]))); @@ -101,7 +102,7 @@ public class PolyFieldTest extends SolrTestCaseJ4 { home = schema.getField("home_ns"); assertNotNull(home); fields = home.createFields(point, 2); - assertEquals(fields.length, 2);//should be 2, since we aren't storing + assertEquals(fields.size(), 2);//should be 2, since we aren't storing home = schema.getField("home_ns"); assertNotNull(home); diff --git a/solr/example/solr/collection1/conf/schema.xml b/solr/example/solr/collection1/conf/schema.xml index caaf5036dce..cc87d86bd24 100755 --- a/solr/example/solr/collection1/conf/schema.xml +++ b/solr/example/solr/collection1/conf/schema.xml @@ -70,6 +70,15 @@ fieldType section indexed: true if this field should be indexed (searchable or sortable) stored: true if this field should be retrievable + docValues: true if this field should have doc values. Doc values are + useful for faceting, grouping, sorting and function queries. Although not + required, doc values will make the index faster to load, more + NRT-friendly and more memory-efficient. They however come with some + limitations: they are currently only supported by StrField, UUIDField + and all Trie*Fields, and depending on the field type, they might + require the field to be single-valued, be required or have a default + value (check the documentation of the field type you're interested in + for more information) multiValued: true if this field may contain multiple values per document omitNorms: (expert) set to true to omit the norms associated with this field (this disables length normalization and index-time @@ -156,6 +165,17 @@ + + @@ -282,7 +302,10 @@ standard package such as org.apache.solr.analysis --> - + @@ -306,6 +329,9 @@ From d540b8e1223a2c1c9a54cb3d75e3d11a5609eab0 Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Sat, 16 Feb 2013 19:17:43 +0000 Subject: [PATCH 07/13] SOLR-4421,SOLR-4165: Fix wait loop to sleep, reduce max wait time, wait min 1 second git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446926 13f79535-47bb-0310-9956-ffa450edef68 --- solr/core/src/java/org/apache/solr/cloud/ZkController.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index f6ae41951f6..8a0cc487c6b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -564,7 +564,7 @@ public final class ZkController { // now wait till the updates are in our state long now = System.currentTimeMillis(); - long timeout = now + 1000 * 300; + long timeout = now + 1000 * 30; boolean foundStates = false; while (System.currentTimeMillis() < timeout) { clusterState = zkStateReader.getClusterState(); @@ -586,8 +586,10 @@ public final class ZkController { if (updatedNodes.size() == 0) { foundStates = true; + Thread.sleep(1000); break; } + Thread.sleep(1000); } if (!foundStates) { log.warn("Timed out waiting to see all nodes published as DOWN in our cluster state."); From 7dd591ba5d674d7a19c40d6d0ef6eac4d283ca5d Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Sat, 16 Feb 2013 19:18:35 +0000 Subject: [PATCH 08/13] tests: raise default so timeout git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446927 13f79535-47bb-0310-9956-ffa450edef68 --- solr/core/src/test-files/solr/solr.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/core/src/test-files/solr/solr.xml b/solr/core/src/test-files/solr/solr.xml index d2300dfe22f..c218d4f9492 100644 --- a/solr/core/src/test-files/solr/solr.xml +++ b/solr/core/src/test-files/solr/solr.xml @@ -30,10 +30,10 @@ --> + distribUpdateConnTimeout="${distribUpdateConnTimeout:15000}" distribUpdateSoTimeout="${distribUpdateSoTimeout:120000}"> - ${socketTimeout:60000} + ${socketTimeout:120000} ${connTimeout:15000} From b800d08400a1d2bbbdcfe2b35330fd217dd52671 Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Sat, 16 Feb 2013 19:20:05 +0000 Subject: [PATCH 09/13] SOLR-4467: Ephemeral directory implementations may not recover correctly because the code to clear the tlog files on startup is off. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446928 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/solr/update/UpdateHandler.java | 7 +++++-- .../java/org/apache/solr/update/UpdateLog.java | 18 +++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java index 5f343658087..0df9ffe32d6 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java @@ -22,6 +22,8 @@ import java.io.File; import java.io.IOException; import java.util.Vector; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrEventListener; @@ -87,13 +89,14 @@ public abstract class UpdateHandler implements SolrInfoMBean { private void clearLog(PluginInfo ulogPluginInfo) { if (ulogPluginInfo == null) return; File tlogDir = UpdateLog.getTlogDir(core, ulogPluginInfo); + log.info("Clearing tlog files, tlogDir=" + tlogDir); if (tlogDir.exists()) { String[] files = UpdateLog.getLogList(tlogDir); for (String file : files) { - File f = new File(file); + File f = new File(tlogDir, file); boolean s = f.delete(); if (!s) { - log.error("Could not remove tlog file:" + f); + throw new SolrException(ErrorCode.SERVER_ERROR, "Could not remove tlog file:" + f.getAbsolutePath()); } } } diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java index fcafb69ad73..ca4e3e9a86f 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java @@ -1398,16 +1398,16 @@ public class UpdateLog implements PluginInfoInitialized { public static File getTlogDir(SolrCore core, PluginInfo info) { String dataDir = (String) info.initArgs.get("dir"); - if (dataDir == null) { - String ulogDir = core.getCoreDescriptor().getUlogDir(); - if (ulogDir != null) { - dataDir = ulogDir; - } - - if (dataDir == null || dataDir.length() == 0) { - dataDir = core.getDataDir(); - } + + String ulogDir = core.getCoreDescriptor().getUlogDir(); + if (ulogDir != null) { + dataDir = ulogDir; } + + if (dataDir == null || dataDir.length() == 0) { + dataDir = core.getDataDir(); + } + return new File(dataDir, TLOG_NAME); } From b6ca8acdaa16776bab04a5f09e48e63a78a04b27 Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Sat, 16 Feb 2013 19:20:57 +0000 Subject: [PATCH 10/13] SOLR-4467: CHANGES entry git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446929 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d908578cd01..88e6d8b60eb 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -134,6 +134,9 @@ Bug Fixes * SOLR-4421,SOLR-4165: On CoreContainer shutdown, all SolrCores should publish their state as DOWN. (Mark Miller, Markus Jelsma) +* SOLR-4467: Ephemeral directory implementations may not recover correctly + because the code to clear the tlog files on startup is off. (Mark Miller) + Optimizations ---------------------- From 902e34af55409417cf88d19b05190300ec4fc0d3 Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Sat, 16 Feb 2013 20:21:33 +0000 Subject: [PATCH 11/13] tests: improve some cloud tests git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446935 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/solr/cloud/ElectionContext.java | 2 +- .../solr/collection1/conf/solrconfig-tlog.xml | 48 ++++++++++++++++++- .../solr/cloud/ChaosMonkeySafeLeaderTest.java | 13 +++-- .../org/apache/solr/cloud/RecoveryZkTest.java | 25 ++++++---- .../solrj/impl/CloudSolrServerTest.java | 4 ++ .../solr/cloud/AbstractDistribZkTestBase.java | 6 ++- .../cloud/AbstractFullDistribZkTestBase.java | 14 ++++++ 7 files changed, 94 insertions(+), 18 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java index 2799f01a02a..75f62672754 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java @@ -294,7 +294,7 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase { Slice slices = zkController.getClusterState().getSlice(collection, shardId); int cnt = 0; - while (true && !isClosed) { + while (true && !isClosed && !cc.isShutDown()) { // wait for everyone to be up if (slices != null) { int found = 0; diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml index c25eef07ce1..dea28e61372 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml @@ -18,10 +18,21 @@ --> + + ${tests.luceneMatchVersion:LUCENE_CURRENT} - + + + + ${solr.hdfs.blockcache.blocksperbank:1024} + + ${solr.data.dir:} + + ${solr.lock.type:native} + + @@ -38,12 +49,14 @@ true + + - + ${solr.ulog.dir:} @@ -72,4 +85,35 @@ + + + + regex_dup_A_s + x + x_x + + + + regex_dup_B_s + x + x_x + + + + + + + + regex_dup_A_s + x + x_x + + + regex_dup_B_s + x + x_x + + + + diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java index 6253defd190..0a5ad55301b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java @@ -20,7 +20,6 @@ package org.apache.solr.cloud; import java.util.ArrayList; import java.util.List; -import org.apache.lucene.util.LuceneTestCase.BadApple; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.common.SolrInputDocument; @@ -37,8 +36,7 @@ import org.junit.BeforeClass; @Slow public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase { - private static final int BASE_RUN_LENGTH = 120000; - private static final int RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", Integer.toString(BASE_RUN_LENGTH))); + private static final Integer RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1")); @BeforeClass public static void beforeSuperClass() { @@ -104,7 +102,14 @@ public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase { } chaosMonkey.startTheMonkey(false, 500); - int runLength = RUN_LENGTH; + long runLength; + if (RUN_LENGTH != -1) { + runLength = RUN_LENGTH; + } else { + int[] runTimes = new int[] {5000,6000,10000,15000,15000,30000,30000,45000,90000,120000}; + runLength = runTimes[random().nextInt(runTimes.length - 1)]; + } + Thread.sleep(runLength); chaosMonkey.stopTheMonkey(); diff --git a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java index 53b0acbdd95..abac858985c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java @@ -50,30 +50,37 @@ public class RecoveryZkTest extends AbstractFullDistribZkTestBase { // start a couple indexing threads - indexThread = new StopableIndexingThread(0, true); + int[] maxDocList = new int[] {300, 700, 1200, 1350, 5000, 15000}; + + int maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)]; + + indexThread = new StopableIndexingThread(0, true, maxDoc); indexThread.start(); - indexThread2 = new StopableIndexingThread(10000, true); + indexThread2 = new StopableIndexingThread(10000, true, maxDoc); indexThread2.start(); // give some time to index... - Thread.sleep(atLeast(2000)); - + int[] waitTimes = new int[] {2000, 3000, 5000}; + Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]); + // bring shard replica down JettySolrRunner replica = chaosMonkey.stopShard("shard1", 1).jetty; // wait a moment - lets allow some docs to be indexed so replication time is non 0 - Thread.sleep(atLeast(2000)); + Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]); // bring shard replica up replica.start(); // make sure replication can start - Thread.sleep(1500); + Thread.sleep(3000); ZkStateReader zkStateReader = cloudClient.getZkStateReader(); - waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, false, true); + + // give some time for replication to complete + Thread.sleep(5000); // stop indexing threads indexThread.safeStop(); @@ -86,12 +93,10 @@ public class RecoveryZkTest extends AbstractFullDistribZkTestBase { waitForThingsToLevelOut(30); - Thread.sleep(1000); + Thread.sleep(2000); waitForThingsToLevelOut(30); - Thread.sleep(5000); - waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, false, true); // test that leader and replica have same doc count diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java index 3e1ade83d15..32a84f373c5 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java @@ -52,6 +52,10 @@ public class CloudSolrServerTest extends AbstractFullDistribZkTestBase { } + protected String getCloudSolrConfig() { + return "solrconfig.xml"; + } + @Override public String getSolrHome() { return SOLR_HOME; diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java index f28c5989fda..1b6c8d1c39d 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java @@ -67,13 +67,17 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes String schema = getSchemaFile(); if (schema == null) schema = "schema.xml"; - AbstractZkTestCase.buildZooKeeper(zkServer.getZkHost(), zkServer.getZkAddress(), "solrconfig.xml", schema); + AbstractZkTestCase.buildZooKeeper(zkServer.getZkHost(), zkServer.getZkAddress(), getCloudSolrConfig(), schema); // set some system properties for use by tests System.setProperty("solr.test.sys.prop1", "propone"); System.setProperty("solr.test.sys.prop2", "proptwo"); } + protected String getCloudSolrConfig() { + return "solrconfig-tlog.xml"; + } + @Override protected void createServers(int numShards) throws Exception { // give everyone there own solrhome diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java index 09f08aac854..4e6c9ea05e8 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java @@ -1140,21 +1140,34 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes protected final List deletes = new ArrayList(); protected final AtomicInteger fails = new AtomicInteger(); protected boolean doDeletes; + private int numCycles; public StopableIndexingThread(int startI, boolean doDeletes) { + this(startI, doDeletes, -1); + } + + public StopableIndexingThread(int startI, boolean doDeletes, int numCycles) { super("StopableIndexingThread"); this.startI = startI; this.doDeletes = doDeletes; + this.numCycles = numCycles; setDaemon(true); } @Override public void run() { int i = startI; + int numDone = 0; int numDeletes = 0; int numAdds = 0; while (true && !stop) { + if (numCycles != -1) { + if (numDone > numCycles) { + break; + } + } + ++numDone; ++i; boolean addFailed = false; @@ -1202,6 +1215,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes @Override public void safeStop() { + System.out.println("safe stop:"); stop = true; } From 2d60ca9bbd154a974a893dce7005577a9993364e Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Sat, 16 Feb 2013 22:11:55 +0000 Subject: [PATCH 12/13] Update forbiddenapis to 1.2 git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446961 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/common-build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/common-build.xml b/lucene/common-build.xml index a85629663a1..d8f024c2b19 100644 --- a/lucene/common-build.xml +++ b/lucene/common-build.xml @@ -1930,7 +1930,7 @@ ${tests-output}/junit4-*.suites - per-JVM executed suites - From 534d8b170eba9b6cf3b1a6bc280c782488f6bfa8 Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Sun, 17 Feb 2013 00:22:57 +0000 Subject: [PATCH 13/13] tests: improve cm tests git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1446981 13f79535-47bb-0310-9956-ffa450edef68 --- .../solr/cloud/ChaosMonkeyNothingIsSafeTest.java | 15 +++++++++++---- .../solr/cloud/ChaosMonkeySafeLeaderTest.java | 9 +++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java index da55de72820..3620b8efe39 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java @@ -46,7 +46,7 @@ import org.slf4j.LoggerFactory; public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase { public static Logger log = LoggerFactory.getLogger(ChaosMonkeyNothingIsSafeTest.class); - private static final int BASE_RUN_LENGTH = 60000; + private static final Integer RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1")); @BeforeClass public static void beforeSuperClass() { @@ -138,8 +138,15 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase } chaosMonkey.startTheMonkey(true, 10000); - //int runLength = atLeast(BASE_RUN_LENGTH); - int runLength = BASE_RUN_LENGTH; + + long runLength; + if (RUN_LENGTH != -1) { + runLength = RUN_LENGTH; + } else { + int[] runTimes = new int[] {5000,6000,10000,15000,15000,30000,30000,45000,90000,120000}; + runLength = runTimes[random().nextInt(runTimes.length - 1)]; + } + try { Thread.sleep(runLength); } finally { @@ -172,7 +179,7 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase // make sure we again have leaders for each shard for (int j = 1; j < sliceCount; j++) { - zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000); + zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000); } commit(); diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java index 0a5ad55301b..7b498e912da 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java @@ -109,10 +109,11 @@ public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase { int[] runTimes = new int[] {5000,6000,10000,15000,15000,30000,30000,45000,90000,120000}; runLength = runTimes[random().nextInt(runTimes.length - 1)]; } - - Thread.sleep(runLength); - - chaosMonkey.stopTheMonkey(); + try { + Thread.sleep(runLength); + } finally { + chaosMonkey.stopTheMonkey(); + } for (StopableIndexingThread indexThread : threads) { indexThread.safeStop();