From af3a19574e6242e1b29f2fb8861bbcc977f23435 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 28 Jan 2016 05:01:15 -0500 Subject: [PATCH 01/21] LUCENE-6998: fix a couple places to better detect truncated index files; improve corruption testing --- lucene/CHANGES.txt | 3 + .../org/apache/lucene/codecs/CodecUtil.java | 4 +- .../lucene50/Lucene50CompoundReader.java | 14 ++ .../codecs/lucene60/Lucene60PointFormat.java | 3 +- .../codecs/lucene60/Lucene60PointReader.java | 64 ++++++--- .../codecs/lucene60/Lucene60PointWriter.java | 8 +- .../index/TestAllFilesCheckIndexHeader.java | 132 ++++++++++++++++++ .../index/TestAllFilesDetectTruncation.java | 127 +++++++++++++++++ .../index/TestAllFilesHaveChecksumFooter.java | 24 ++-- .../index/TestAllFilesHaveCodecHeader.java | 31 +--- .../lucene/index/TestSwappedIndexFiles.java | 127 +++++++++++++++++ 11 files changed, 475 insertions(+), 62 deletions(-) create mode 100644 lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java create mode 100644 lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java create mode 100644 lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 36a667495ef..ad22c43930d 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -211,6 +211,9 @@ Bug Fixes * LUCENE-6984: SpanMultiTermQueryWrapper no longer modifies its wrapped query. (Alan Woodward, Adrien Grand) +* LUCENE-6998: Fix a couple places to better detect truncated index files + as corruption. (Robert Muir, Mike McCandless) + Other * LUCENE-6924: Upgrade randomizedtesting to 2.3.2. (Dawid Weiss) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java index 1bc2f40a9d1..9ff3cdbc464 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java @@ -17,7 +17,6 @@ package org.apache.lucene.codecs; * limitations under the License. */ - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -397,6 +396,9 @@ public final class CodecUtil { * @throws IOException if the footer is invalid */ public static long retrieveChecksum(IndexInput in) throws IOException { + if (in.length() < footerLength()) { + throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), in); + } in.seek(in.length() - footerLength()); validateFooter(in); return readCRC(in); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java index 57df8aa4d93..b1cfc5dd36f 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundReader.java @@ -68,6 +68,13 @@ final class Lucene50CompoundReader extends Directory { String entriesFileName = IndexFileNames.segmentFileName(segmentName, "", Lucene50CompoundFormat.ENTRIES_EXTENSION); this.entries = readEntries(si.getId(), directory, entriesFileName); boolean success = false; + + long expectedLength = CodecUtil.indexHeaderLength(Lucene50CompoundFormat.DATA_CODEC, ""); + for(Map.Entry ent : entries.entrySet()) { + expectedLength += ent.getValue().length; + } + expectedLength += CodecUtil.footerLength(); + handle = directory.openInput(dataFileName, context); try { CodecUtil.checkIndexHeader(handle, Lucene50CompoundFormat.DATA_CODEC, version, version, si.getId(), ""); @@ -77,6 +84,13 @@ final class Lucene50CompoundReader extends Directory { // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption // such as file truncation. CodecUtil.retrieveChecksum(handle); + + // We also validate length, because e.g. if you strip 16 bytes off the .cfs we otherwise + // would not detect it: + if (handle.length() != expectedLength) { + throw new CorruptIndexException("length should be " + expectedLength + " bytes, but is " + handle.length() + " instead", handle); + } + success = true; } finally { if (!success) { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointFormat.java index 61ce8fbdc4d..beec4bd340c 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointFormat.java @@ -73,7 +73,8 @@ import org.apache.lucene.index.SegmentWriteState; public final class Lucene60PointFormat extends PointFormat { - static final String CODEC_NAME = "Lucene60PointFormat"; + static final String DATA_CODEC_NAME = "Lucene60PointFormatData"; + static final String META_CODEC_NAME = "Lucene60PointFormatMeta"; /** * Filename extension for the leaf blocks diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointReader.java index 2e2bddbe80f..2d18019e553 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointReader.java @@ -1,6 +1,5 @@ package org.apache.lucene.codecs.lucene60; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -48,38 +47,69 @@ public class Lucene60PointReader extends PointReader implements Closeable { /** Sole constructor */ public Lucene60PointReader(SegmentReadState readState) throws IOException { this.readState = readState; - String dataFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, - readState.segmentSuffix, - Lucene60PointFormat.DATA_EXTENSION); - dataIn = readState.directory.openInput(dataFileName, readState.context); + + String indexFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, readState.segmentSuffix, Lucene60PointFormat.INDEX_EXTENSION); - boolean success = false; + Map fieldToFileOffset = new HashMap<>(); // Read index file try (ChecksumIndexInput indexIn = readState.directory.openChecksumInput(indexFileName, readState.context)) { - CodecUtil.checkIndexHeader(indexIn, - Lucene60PointFormat.CODEC_NAME, - Lucene60PointFormat.INDEX_VERSION_START, - Lucene60PointFormat.INDEX_VERSION_START, + Throwable priorE = null; + try { + CodecUtil.checkIndexHeader(indexIn, + Lucene60PointFormat.META_CODEC_NAME, + Lucene60PointFormat.INDEX_VERSION_START, + Lucene60PointFormat.INDEX_VERSION_START, + readState.segmentInfo.getId(), + readState.segmentSuffix); + int count = indexIn.readVInt(); + for(int i=0;i ent : fieldToFileOffset.entrySet()) { + int fieldNumber = ent.getKey(); + long fp = ent.getValue(); dataIn.seek(fp); BKDReader reader = new BKDReader(dataIn); readers.put(fieldNumber, reader); - //reader.verify(readState.segmentInfo.maxDoc()); } - CodecUtil.checkFooter(indexIn); + success = true; } finally { if (success == false) { - IOUtils.closeWhileHandlingException(dataIn); + IOUtils.closeWhileHandlingException(this); } } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java index 318d665ee2e..e7104b6a586 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java @@ -27,12 +27,12 @@ import java.util.Map; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PointReader; import org.apache.lucene.codecs.PointWriter; -import org.apache.lucene.index.PointValues.IntersectVisitor; -import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.PointValues.IntersectVisitor; +import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.IOUtils; @@ -62,7 +62,7 @@ public class Lucene60PointWriter extends PointWriter implements Closeable { boolean success = false; try { CodecUtil.writeIndexHeader(dataOut, - Lucene60PointFormat.CODEC_NAME, + Lucene60PointFormat.DATA_CODEC_NAME, Lucene60PointFormat.DATA_VERSION_CURRENT, writeState.segmentInfo.getId(), writeState.segmentSuffix); @@ -184,7 +184,7 @@ public class Lucene60PointWriter extends PointWriter implements Closeable { // Write index file try (IndexOutput indexOut = writeState.directory.createOutput(indexFileName, writeState.context)) { CodecUtil.writeIndexHeader(indexOut, - Lucene60PointFormat.CODEC_NAME, + Lucene60PointFormat.META_CODEC_NAME, Lucene60PointFormat.INDEX_VERSION_CURRENT, writeState.segmentInfo.getId(), writeState.segmentSuffix); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java new file mode 100644 index 00000000000..9dffaa8df6c --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java @@ -0,0 +1,132 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.EOFException; +import java.io.IOException; +import java.util.Collections; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.store.BaseDirectoryWrapper; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.util.LineFileDocs; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +/** + * Test that a plain default detects broken index headers early (on opening a reader). + */ +public class TestAllFilesCheckIndexHeader extends LuceneTestCase { + public void test() throws Exception { + Directory dir = newDirectory(); + + if (dir instanceof MockDirectoryWrapper) { + // otherwise we can have unref'd files left in the index that won't be visited when opening a reader and lead to scary looking false failures: + ((MockDirectoryWrapper) dir).setEnableVirusScanner(false); + } + + IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); + conf.setCodec(TestUtil.getDefaultCodec()); + + // Disable CFS 80% of the time so we can truncate individual files, but the other 20% of the time we test truncation of .cfs/.cfe too: + if (random().nextInt(5) != 1) { + conf.setUseCompoundFile(false); + conf.getMergePolicy().setNoCFSRatio(0.0); + } + + RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf); + // Use LineFileDocs so we (hopefully) get most Lucene features + // tested, e.g. IntPoint was recently added to it: + LineFileDocs docs = new LineFileDocs(random()); + for (int i = 0; i < 100; i++) { + riw.addDocument(docs.nextDoc()); + if (random().nextInt(7) == 0) { + riw.commit(); + } + if (random().nextInt(20) == 0) { + riw.deleteDocuments(new Term("docid", Integer.toString(i))); + } + if (random().nextInt(15) == 0) { + riw.updateNumericDocValue(new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i)); + } + } + + if (TEST_NIGHTLY == false) { + riw.forceMerge(1); + } + riw.close(); + checkIndexHeader(dir); + dir.close(); + } + + private void checkIndexHeader(Directory dir) throws IOException { + for(String name : dir.listAll()) { + checkOneFile(dir, name); + } + } + + private void checkOneFile(Directory dir, String victim) throws IOException { + try (BaseDirectoryWrapper dirCopy = newDirectory()) { + dirCopy.setCheckIndexOnClose(false); + long victimLength = dir.fileLength(victim); + int wrongBytes = TestUtil.nextInt(random(), 1, (int) Math.min(100, victimLength)); + assert victimLength > 0; + + if (VERBOSE) { + System.out.println("TEST: now break file " + victim + " by randomizing first " + wrongBytes + " of " + victimLength); + } + + for(String name : dir.listAll()) { + if (name.equals(victim) == false) { + dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT); + } else { + try(IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT); + IndexInput in = dir.openInput(name, IOContext.DEFAULT)) { + // keeps same file length, but replaces the first wrongBytes with random bytes: + byte[] bytes = new byte[wrongBytes]; + random().nextBytes(bytes); + out.writeBytes(bytes, 0, bytes.length); + in.seek(wrongBytes); + out.copyBytes(in, victimLength - wrongBytes); + } + } + dirCopy.sync(Collections.singleton(name)); + } + + try { + // NOTE: we .close so that if the test fails (truncation not detected) we don't also get all these confusing errors about open files: + DirectoryReader.open(dirCopy).close(); + fail("wrong bytes not detected after randomizing first " + wrongBytes + " bytes out of " + victimLength + " for file " + victim); + } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) { + // expected + } + + // CheckIndex should also fail: + try { + TestUtil.checkIndex(dirCopy, true, true); + fail("wrong bytes not detected after randomizing first " + wrongBytes + " bytes out of " + victimLength + " for file " + victim); + } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) { + // expected + } + } + } +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java new file mode 100644 index 00000000000..bda5857a7a7 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java @@ -0,0 +1,127 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.EOFException; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.store.BaseDirectoryWrapper; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.util.LineFileDocs; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +/** + * Test that a plain default detects index file truncation early (on opening a reader). + */ +public class TestAllFilesDetectTruncation extends LuceneTestCase { + public void test() throws Exception { + Directory dir = newDirectory(); + + if (dir instanceof MockDirectoryWrapper) { + // otherwise we can have unref'd files left in the index that won't be visited when opening a reader and lead to scary looking false failures: + ((MockDirectoryWrapper) dir).setEnableVirusScanner(false); + } + + IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); + conf.setCodec(TestUtil.getDefaultCodec()); + + // Disable CFS 80% of the time so we can truncate individual files, but the other 20% of the time we test truncation of .cfs/.cfe too: + if (random().nextInt(5) != 1) { + conf.setUseCompoundFile(false); + conf.getMergePolicy().setNoCFSRatio(0.0); + } + + RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf); + // Use LineFileDocs so we (hopefully) get most Lucene features + // tested, e.g. IntPoint was recently added to it: + LineFileDocs docs = new LineFileDocs(random()); + for (int i = 0; i < 100; i++) { + riw.addDocument(docs.nextDoc()); + if (random().nextInt(7) == 0) { + riw.commit(); + } + if (random().nextInt(20) == 0) { + riw.deleteDocuments(new Term("docid", Integer.toString(i))); + } + if (random().nextInt(15) == 0) { + riw.updateNumericDocValue(new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i)); + } + } + if (TEST_NIGHTLY == false) { + riw.forceMerge(1); + } + riw.close(); + checkTruncation(dir); + dir.close(); + } + + private void checkTruncation(Directory dir) throws IOException { + for(String name : dir.listAll()) { + truncateOneFile(dir, name); + } + } + + private void truncateOneFile(Directory dir, String victim) throws IOException { + try (BaseDirectoryWrapper dirCopy = newDirectory()) { + dirCopy.setCheckIndexOnClose(false); + long victimLength = dir.fileLength(victim); + int lostBytes = TestUtil.nextInt(random(), 1, (int) Math.min(100, victimLength)); + assert victimLength > 0; + + if (VERBOSE) { + System.out.println("TEST: now truncate file " + victim + " by removing " + lostBytes + " of " + victimLength + " bytes"); + } + + for(String name : dir.listAll()) { + if (name.equals(victim) == false) { + dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT); + } else { + try(IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT); + IndexInput in = dir.openInput(name, IOContext.DEFAULT)) { + out.copyBytes(in, victimLength - lostBytes); + } + } + dirCopy.sync(Collections.singleton(name)); + } + + try { + // NOTE: we .close so that if the test fails (truncation not detected) we don't also get all these confusing errors about open files: + DirectoryReader.open(dirCopy).close(); + fail("truncation not detected after removing " + lostBytes + " bytes out of " + victimLength + " for file " + victim); + } catch (CorruptIndexException | EOFException e) { + // expected + } + + // CheckIndex should also fail: + try { + TestUtil.checkIndex(dirCopy, true, true); + fail("truncation not detected after removing " + lostBytes + " bytes out of " + victimLength + " for file " + victim); + } catch (CorruptIndexException | EOFException e) { + // expected + } + } + } +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java index 66eb343f7cf..710d20f86f5 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java @@ -21,11 +21,9 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -38,23 +36,19 @@ public class TestAllFilesHaveChecksumFooter extends LuceneTestCase { IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); conf.setCodec(TestUtil.getDefaultCodec()); RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf); - Document doc = new Document(); - // these fields should sometimes get term vectors, etc - Field idField = newStringField("id", "", Field.Store.NO); - Field bodyField = newTextField("body", "", Field.Store.NO); - Field dvField = new NumericDocValuesField("dv", 5); - doc.add(idField); - doc.add(bodyField); - doc.add(dvField); + // Use LineFileDocs so we (hopefully) get most Lucene features + // tested, e.g. IntPoint was recently added to it: + LineFileDocs docs = new LineFileDocs(random()); for (int i = 0; i < 100; i++) { - idField.setStringValue(Integer.toString(i)); - bodyField.setStringValue(TestUtil.randomUnicodeString(random())); - riw.addDocument(doc); + riw.addDocument(docs.nextDoc()); if (random().nextInt(7) == 0) { riw.commit(); } if (random().nextInt(20) == 0) { - riw.deleteDocuments(new Term("id", Integer.toString(i))); + riw.deleteDocuments(new Term("docid", Integer.toString(i))); + } + if (random().nextInt(15) == 0) { + riw.updateNumericDocValue(new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i)); } } riw.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java index c2b515db718..d7260195283 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java @@ -23,13 +23,9 @@ import java.util.Map; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.document.TextField; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -43,32 +39,19 @@ public class TestAllFilesHaveCodecHeader extends LuceneTestCase { IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); conf.setCodec(TestUtil.getDefaultCodec()); RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf); - Document doc = new Document(); - Field idField = newStringField("id", "", Field.Store.YES); - Field bodyField = newTextField("body", "", Field.Store.YES); - FieldType vectorsType = new FieldType(TextField.TYPE_STORED); - vectorsType.setStoreTermVectors(true); - vectorsType.setStoreTermVectorPositions(true); - Field vectorsField = new Field("vectors", "", vectorsType); - Field dvField = new NumericDocValuesField("dv", 5); - doc.add(idField); - doc.add(bodyField); - doc.add(vectorsField); - doc.add(dvField); + // Use LineFileDocs so we (hopefully) get most Lucene features + // tested, e.g. IntPoint was recently added to it: + LineFileDocs docs = new LineFileDocs(random()); for (int i = 0; i < 100; i++) { - idField.setStringValue(Integer.toString(i)); - bodyField.setStringValue(TestUtil.randomUnicodeString(random())); - dvField.setLongValue(random().nextInt(5)); - vectorsField.setStringValue(TestUtil.randomUnicodeString(random())); - riw.addDocument(doc); + riw.addDocument(docs.nextDoc()); if (random().nextInt(7) == 0) { riw.commit(); } if (random().nextInt(20) == 0) { - riw.deleteDocuments(new Term("id", Integer.toString(i))); + riw.deleteDocuments(new Term("docid", Integer.toString(i))); } if (random().nextInt(15) == 0) { - riw.updateNumericDocValue(new Term("id"), "dv", Long.valueOf(i)); + riw.updateNumericDocValue(new Term("docid", Integer.toString(i)), "docid_intDV", Long.valueOf(i)); } } riw.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java new file mode 100644 index 00000000000..008e9a69e1f --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java @@ -0,0 +1,127 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.EOFException; +import java.io.IOException; +import java.util.Collections; +import java.util.Random; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.store.BaseDirectoryWrapper; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.util.LineFileDocs; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +/** + * Test that the same file name, but from a different index, is detected as foreign. + */ +public class TestSwappedIndexFiles extends LuceneTestCase { + public void test() throws Exception { + Directory dir1 = newDirectory(); + Directory dir2 = newDirectory(); + + if (dir1 instanceof MockDirectoryWrapper) { + // otherwise we can have unref'd files left in the index that won't be visited when opening a reader and lead to scary looking false failures: + ((MockDirectoryWrapper) dir1).setEnableVirusScanner(false); + } + if (dir2 instanceof MockDirectoryWrapper) { + // otherwise we can have unref'd files left in the index that won't be visited when opening a reader and lead to scary looking false failures: + ((MockDirectoryWrapper) dir2).setEnableVirusScanner(false); + } + + // Disable CFS 80% of the time so we can truncate individual files, but the other 20% of the time we test truncation of .cfs/.cfe too: + boolean useCFS = random().nextInt(5) == 1; + + // Use LineFileDocs so we (hopefully) get most Lucene features + // tested, e.g. IntPoint was recently added to it: + LineFileDocs docs = new LineFileDocs(random()); + Document doc = docs.nextDoc(); + long seed = random().nextLong(); + + indexOneDoc(seed, dir1, doc, useCFS); + indexOneDoc(seed, dir2, doc, useCFS); + + swapFiles(dir1, dir2); + dir1.close(); + dir2.close(); + } + + private void indexOneDoc(long seed, Directory dir, Document doc, boolean useCFS) throws IOException { + Random random = new Random(seed); + IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random)); + conf.setCodec(TestUtil.getDefaultCodec()); + + if (useCFS == false) { + conf.setUseCompoundFile(false); + conf.getMergePolicy().setNoCFSRatio(0.0); + } else { + conf.setUseCompoundFile(true); + conf.getMergePolicy().setNoCFSRatio(1.0); + } + + RandomIndexWriter w = new RandomIndexWriter(random, dir, conf); + w.addDocument(doc); + w.close(); + } + + private void swapFiles(Directory dir1, Directory dir2) throws IOException { + for(String name : dir1.listAll()) { + if (name.equals(IndexWriter.WRITE_LOCK_NAME)) { + continue; + } + swapOneFile(dir1, dir2, name); + } + } + + private void swapOneFile(Directory dir1, Directory dir2, String victim) throws IOException { + try (BaseDirectoryWrapper dirCopy = newDirectory()) { + dirCopy.setCheckIndexOnClose(false); + + // Copy all files from dir1 to dirCopy, except victim which we copy from dir2: + for(String name : dir1.listAll()) { + if (name.equals(victim) == false) { + dirCopy.copyFrom(dir1, name, name, IOContext.DEFAULT); + } else { + dirCopy.copyFrom(dir2, name, name, IOContext.DEFAULT); + } + dirCopy.sync(Collections.singleton(name)); + } + + try { + // NOTE: we .close so that if the test fails (truncation not detected) we don't also get all these confusing errors about open files: + DirectoryReader.open(dirCopy).close(); + fail("wrong file " + victim + " not detected"); + } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) { + // expected + } + + // CheckIndex should also fail: + try { + TestUtil.checkIndex(dirCopy, true, true); + fail("wrong file " + victim + " not detected"); + } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) { + // expected + } + } + } +} From 791ddc627be6281f878cc2320247299582906757 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 28 Jan 2016 11:02:24 -0500 Subject: [PATCH 02/21] LUCENE-6999: break out PointWriter.finish from .close to avoid leaking file handles on exception --- .../simpletext/SimpleTextPointWriter.java | 6 +- .../org/apache/lucene/codecs/PointWriter.java | 4 ++ .../codecs/lucene60/Lucene60PointWriter.java | 69 ++++++++++--------- .../lucene/index/DefaultIndexingChain.java | 3 + .../apache/lucene/index/TestAddIndexes.java | 14 ++++ .../apache/lucene/index/TestAtomicUpdate.java | 1 + .../lucene/index/TestCodecHoldsOpenFiles.java | 4 ++ .../index/TestIndexWriterExceptions2.java | 2 + .../index/TestIndexWriterOnDiskFull.java | 3 + .../index/TestIndexWriterOnVMError.java | 2 + .../asserting/AssertingPointFormat.java | 5 ++ 11 files changed, 81 insertions(+), 32 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointWriter.java index fc533da39a9..8d5f107e508 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointWriter.java @@ -196,10 +196,14 @@ class SimpleTextPointWriter extends PointWriter { SimpleTextUtil.writeNewline(out); } + @Override + public void finish() throws IOException { + SimpleTextUtil.writeChecksum(dataOut, scratch); + } + @Override public void close() throws IOException { if (dataOut != null) { - SimpleTextUtil.writeChecksum(dataOut, scratch); dataOut.close(); dataOut = null; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/PointWriter.java index 8e946018799..e53a8b0df68 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/PointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/PointWriter.java @@ -126,5 +126,9 @@ public abstract class PointWriter implements Closeable { mergeOneField(mergeState, fieldInfo); } } + finish(); } + + /** Called once at the end before close */ + public abstract void finish() throws IOException; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java index e7104b6a586..01491fdcdc8 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java @@ -47,7 +47,7 @@ public class Lucene60PointWriter extends PointWriter implements Closeable { final SegmentWriteState writeState; final int maxPointsInLeafNode; final double maxMBSortInHeap; - private boolean closed; + private boolean finished; /** Full constructor */ public Lucene60PointWriter(SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap) throws IOException { @@ -168,38 +168,45 @@ public class Lucene60PointWriter extends PointWriter implements Closeable { mergeOneField(mergeState, fieldInfo); } } - } - } + } + + finish(); + } + + @Override + public void finish() throws IOException { + if (finished) { + throw new IllegalStateException("already finished"); + } + finished = true; + CodecUtil.writeFooter(dataOut); + + String indexFileName = IndexFileNames.segmentFileName(writeState.segmentInfo.name, + writeState.segmentSuffix, + Lucene60PointFormat.INDEX_EXTENSION); + // Write index file + try (IndexOutput indexOut = writeState.directory.createOutput(indexFileName, writeState.context)) { + CodecUtil.writeIndexHeader(indexOut, + Lucene60PointFormat.META_CODEC_NAME, + Lucene60PointFormat.INDEX_VERSION_CURRENT, + writeState.segmentInfo.getId(), + writeState.segmentSuffix); + int count = indexFPs.size(); + indexOut.writeVInt(count); + for(Map.Entry ent : indexFPs.entrySet()) { + FieldInfo fieldInfo = writeState.fieldInfos.fieldInfo(ent.getKey()); + if (fieldInfo == null) { + throw new IllegalStateException("wrote field=\"" + ent.getKey() + "\" but that field doesn't exist in FieldInfos"); + } + indexOut.writeVInt(fieldInfo.number); + indexOut.writeVLong(ent.getValue()); + } + CodecUtil.writeFooter(indexOut); + } + } @Override public void close() throws IOException { - if (closed == false) { - CodecUtil.writeFooter(dataOut); - dataOut.close(); - closed = true; - - String indexFileName = IndexFileNames.segmentFileName(writeState.segmentInfo.name, - writeState.segmentSuffix, - Lucene60PointFormat.INDEX_EXTENSION); - // Write index file - try (IndexOutput indexOut = writeState.directory.createOutput(indexFileName, writeState.context)) { - CodecUtil.writeIndexHeader(indexOut, - Lucene60PointFormat.META_CODEC_NAME, - Lucene60PointFormat.INDEX_VERSION_CURRENT, - writeState.segmentInfo.getId(), - writeState.segmentSuffix); - int count = indexFPs.size(); - indexOut.writeVInt(count); - for(Map.Entry ent : indexFPs.entrySet()) { - FieldInfo fieldInfo = writeState.fieldInfos.fieldInfo(ent.getKey()); - if (fieldInfo == null) { - throw new IllegalStateException("wrote field=\"" + ent.getKey() + "\" but that field doesn't exist in FieldInfos"); - } - indexOut.writeVInt(fieldInfo.number); - indexOut.writeVLong(ent.getValue()); - } - CodecUtil.writeFooter(indexOut); - } - } + dataOut.close(); } } diff --git a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java index 23dc6f66746..788fa2933ac 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java +++ b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java @@ -152,6 +152,9 @@ final class DefaultIndexingChain extends DocConsumer { perField = perField.next; } } + if (pointWriter != null) { + pointWriter.finish(); + } success = true; } finally { if (success) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java index 6ce76a594da..ca6808f80a5 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java @@ -32,6 +32,8 @@ import org.apache.lucene.codecs.memory.MemoryPostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -169,6 +171,8 @@ public class TestAddIndexes extends LuceneTestCase { Document doc = new Document(); doc.add(newStringField("id", "" + (i % 10), Field.Store.NO)); doc.add(newTextField("content", "bbb " + i, Field.Store.NO)); + doc.add(new IntPoint("doc", i)); + doc.add(new NumericDocValuesField("dv", i)); writer.updateDocument(new Term("id", "" + (i%10)), doc); } // Deletes one of the 10 added docs, leaving 9: @@ -202,6 +206,8 @@ public class TestAddIndexes extends LuceneTestCase { Document doc = new Document(); doc.add(newStringField("id", "" + (i % 10), Field.Store.NO)); doc.add(newTextField("content", "bbb " + i, Field.Store.NO)); + doc.add(new IntPoint("doc", i)); + doc.add(new NumericDocValuesField("dv", i)); writer.updateDocument(new Term("id", "" + (i%10)), doc); } @@ -238,6 +244,8 @@ public class TestAddIndexes extends LuceneTestCase { Document doc = new Document(); doc.add(newStringField("id", "" + (i % 10), Field.Store.NO)); doc.add(newTextField("content", "bbb " + i, Field.Store.NO)); + doc.add(new IntPoint("doc", i)); + doc.add(new NumericDocValuesField("dv", i)); writer.updateDocument(new Term("id", "" + (i%10)), doc); } @@ -510,6 +518,8 @@ public class TestAddIndexes extends LuceneTestCase { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); doc.add(newTextField("content", "aaa", Field.Store.NO)); + doc.add(new IntPoint("doc", i)); + doc.add(new NumericDocValuesField("dv", i)); writer.addDocument(doc); } } @@ -518,6 +528,8 @@ public class TestAddIndexes extends LuceneTestCase { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); doc.add(newTextField("content", "bbb", Field.Store.NO)); + doc.add(new IntPoint("doc", i)); + doc.add(new NumericDocValuesField("dv", i)); writer.addDocument(doc); } } @@ -1001,6 +1013,8 @@ public class TestAddIndexes extends LuceneTestCase { Document doc = new Document(); doc.add(newTextField("content", "aaa", Field.Store.NO)); doc.add(newTextField("id", "" + (docStart + i), Field.Store.YES)); + doc.add(new IntPoint("doc", i)); + doc.add(new NumericDocValuesField("dv", i)); writer.addDocument(doc); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java b/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java index 1bede7a1b38..1eba37a8278 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java @@ -80,6 +80,7 @@ public class TestAtomicUpdate extends LuceneTestCase { Document d = new Document(); d.add(new StringField("id", Integer.toString(i), Field.Store.YES)); d.add(new TextField("contents", English.intToEnglish(i+10*count), Field.Store.NO)); + d.add(new IntPoint("doc", i)); writer.updateDocument(new Term("id", Integer.toString(i)), d); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java index 546a6bd76f6..0686048b71b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java @@ -20,6 +20,8 @@ package org.apache.lucene.index; import java.io.IOException; import org.apache.lucene.document.Document; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.TextField; import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; @@ -35,6 +37,8 @@ public class TestCodecHoldsOpenFiles extends LuceneTestCase { for(int i=0;i 0) { // single doc diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java index cff2b83b4bf..8163f8e595b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java @@ -24,6 +24,7 @@ import org.apache.lucene.codecs.LiveDocsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -572,6 +573,7 @@ public class TestIndexWriterOnDiskFull extends LuceneTestCase { Document doc = new Document(); doc.add(newTextField("content", "aaa", Field.Store.NO)); doc.add(new NumericDocValuesField("numericdv", 1)); + doc.add(new IntPoint("point", 1)); writer.addDocument(doc); } @@ -580,6 +582,7 @@ public class TestIndexWriterOnDiskFull extends LuceneTestCase { doc.add(newTextField("content", "aaa " + index, Field.Store.NO)); doc.add(newTextField("id", "" + index, Field.Store.NO)); doc.add(new NumericDocValuesField("numericdv", 1)); + doc.add(new IntPoint("point", 1)); writer.addDocument(doc); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnVMError.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnVMError.java index 90371fe08d5..6c62df2bcfd 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnVMError.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnVMError.java @@ -31,6 +31,7 @@ import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; @@ -124,6 +125,7 @@ public class TestIndexWriterOnVMError extends LuceneTestCase { FieldType ft = new FieldType(TextField.TYPE_NOT_STORED); ft.setStoreTermVectors(true); doc.add(newField("text_vectors", TestUtil.randomAnalysisString(random(), 6, true), ft)); + doc.add(new IntPoint("point", random().nextInt())); if (random().nextInt(10) > 0) { // single doc diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java index 7b306d7e10a..173e257cbd6 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPointFormat.java @@ -140,6 +140,11 @@ public final class AssertingPointFormat extends PointFormat { in.merge(mergeState); } + @Override + public void finish() throws IOException { + in.finish(); + } + @Override public void close() throws IOException { in.close(); From 79e384bac5604fe999bd7293ea568b8ad7a014c7 Mon Sep 17 00:00:00 2001 From: jbernste Date: Thu, 28 Jan 2016 11:07:47 -0500 Subject: [PATCH 03/21] SOLR-8602: Implement ResultSetImpl.wasNull() --- .../org/apache/solr/handler/SQLHandler.java | 18 +- .../client/solrj/io/sql/ResultSetImpl.java | 87 +++++++-- .../solr/client/solrj/io/sql/JdbcTest.java | 179 ++++++++++++++++-- 3 files changed, 249 insertions(+), 35 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java index fa1b3921bb8..7bbe7ea470f 100644 --- a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java @@ -112,11 +112,15 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware { throw new Exception("sql parameter cannot be null"); } - TupleStream tupleStream = SQLTupleStreamParser.parse(sql, numWorkers, workerCollection, workerZkhost, - AggregationMode.getMode(mode), includeMetadata); - context.numWorkers = numWorkers; context.setSolrClientCache(StreamHandler.clientCache); - tupleStream.setStreamContext(context); + + TupleStream tupleStream = SQLTupleStreamParser.parse(sql, + numWorkers, + workerCollection, + workerZkhost, + AggregationMode.getMode(mode), + includeMetadata, + context); rsp.add("result-set", new StreamHandler.TimerStream(new ExceptionStream(tupleStream))); } catch(Exception e) { @@ -148,7 +152,8 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware { String workerCollection, String workerZkhost, AggregationMode aggregationMode, - boolean includeMetadata) throws IOException { + boolean includeMetadata, + StreamContext context) throws IOException { SqlParser parser = new SqlParser(); Statement statement = parser.createStatement(sql); @@ -163,12 +168,14 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware { if(aggregationMode == AggregationMode.FACET) { sqlStream = doGroupByWithAggregatesFacets(sqlVistor); } else { + context.numWorkers = numWorkers; sqlStream = doGroupByWithAggregates(sqlVistor, numWorkers, workerCollection, workerZkhost); } } else if(sqlVistor.isDistinct) { if(aggregationMode == AggregationMode.FACET) { sqlStream = doSelectDistinctFacets(sqlVistor); } else { + context.numWorkers = numWorkers; sqlStream = doSelectDistinct(sqlVistor, numWorkers, workerCollection, workerZkhost); } } else { @@ -179,6 +186,7 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware { sqlStream = new MetadataStream(sqlStream, sqlVistor); } + sqlStream.setStreamContext(context); return sqlStream; } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ResultSetImpl.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ResultSetImpl.java index e354300a018..0f4dcf0d106 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ResultSetImpl.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/sql/ResultSetImpl.java @@ -53,6 +53,7 @@ class ResultSetImpl implements ResultSet { private boolean done; private boolean closed; private SQLWarning currentWarning; + private boolean wasLastValueNull; ResultSetImpl(StatementImpl statement) { this.statement = statement; @@ -66,15 +67,11 @@ class ResultSetImpl implements ResultSet { if(isMetadata == null || !isMetadata.equals(true)) { throw new RuntimeException("First tuple is not a metadata tuple"); } - } catch (IOException e) { - throw new RuntimeException("Couldn't get metadata tuple"); - } - try { this.firstTuple = this.solrStream.read(); this.solrStream.pushBack(firstTuple); } catch (IOException e) { - throw new RuntimeException("Couldn't get first tuple."); + throw new RuntimeException("Couldn't read first tuple", e); } this.resultSetMetaData = new ResultSetMetaDataImpl(this); @@ -122,8 +119,7 @@ class ResultSetImpl implements ResultSet { @Override public boolean wasNull() throws SQLException { - // TODO implement logic to check if last value was null - return false; + return this.wasLastValueNull; } @Override @@ -208,24 +204,38 @@ class ResultSetImpl implements ResultSet { @Override public String getString(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); - return tuple.getString(columnLabel); + String value = tuple.getString(columnLabel); + if(value.equals(String.valueOf((Object)null))) { + this.wasLastValueNull = true; + return null; + } + return value; } @Override public boolean getBoolean(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); - return (boolean)getObject(columnLabel); + Object value = getObject(columnLabel); + if(value == null) { + this.wasLastValueNull = true; + return false; + } + return (boolean)value; } @Override public byte getByte(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); Number number = (Number)getObject(columnLabel); if(number == null) { + this.wasLastValueNull = true; return 0; } else { return number.byteValue(); @@ -234,9 +244,12 @@ class ResultSetImpl implements ResultSet { @Override public short getShort(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); + Number number = (Number)getObject(columnLabel); if(number == null) { + this.wasLastValueNull = true; return 0; } else { return number.shortValue(); @@ -245,10 +258,12 @@ class ResultSetImpl implements ResultSet { @Override public int getInt(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); Number number = (Number)getObject(columnLabel); if(number == null) { + this.wasLastValueNull = true; return 0; } else { return number.intValue(); @@ -257,10 +272,12 @@ class ResultSetImpl implements ResultSet { @Override public long getLong(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); Number number = (Number)getObject(columnLabel); if(number == null) { + this.wasLastValueNull = true; return 0L; } else { return number.longValue(); @@ -269,10 +286,12 @@ class ResultSetImpl implements ResultSet { @Override public float getFloat(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); Number number = (Number)getObject(columnLabel); if(number == null) { + this.wasLastValueNull = true; return 0.0F; } else { return number.floatValue(); @@ -281,10 +300,12 @@ class ResultSetImpl implements ResultSet { @Override public double getDouble(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); Number number = (Number)getObject(columnLabel); if(number == null) { + this.wasLastValueNull = true; return 0.0D; } else { return number.doubleValue(); @@ -298,30 +319,54 @@ class ResultSetImpl implements ResultSet { @Override public byte[] getBytes(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); - return (byte[]) getObject(columnLabel); + Object value = getObject(columnLabel); + if(value == null) { + this.wasLastValueNull = true; + return null; + } + return (byte[])value; } @Override public Date getDate(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); - return (Date)getObject(columnLabel); + Object value = getObject(columnLabel); + if(value == null) { + this.wasLastValueNull = true; + return null; + } + return (Date)value; } @Override public Time getTime(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); - return (Time)getObject(columnLabel); + Object value = getObject(columnLabel); + if(value == null) { + this.wasLastValueNull = true; + return null; + } + return (Time)value; } @Override public Timestamp getTimestamp(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); - return (Timestamp)getObject(columnLabel); + Object value = getObject(columnLabel); + if(value == null) { + this.wasLastValueNull = true; + return null; + } + return (Timestamp)value; } @Override @@ -376,9 +421,15 @@ class ResultSetImpl implements ResultSet { @Override public Object getObject(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); - return this.tuple.get(columnLabel); + Object value = this.tuple.get(columnLabel); + if(value == null) { + this.wasLastValueNull = true; + return null; + } + return value; } @Override @@ -403,9 +454,15 @@ class ResultSetImpl implements ResultSet { @Override public BigDecimal getBigDecimal(String columnLabel) throws SQLException { + this.wasLastValueNull = false; checkClosed(); - return (BigDecimal)getObject(columnLabel); + Object value = this.getObject(columnLabel); + if(value == null) { + this.wasLastValueNull = true; + return null; + } + return (BigDecimal)value; } @Override diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java index 02f403f6611..e93cf10928a 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java @@ -92,16 +92,16 @@ public class JdbcTest extends AbstractFullDistribZkTestBase { waitForRecoveriesToFinish(false); - indexr(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1"); - indexr(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2"); - indexr(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3"); - indexr(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4"); - indexr(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5"); - indexr(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6"); - indexr(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7"); - indexr(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8"); - indexr(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9"); - indexr(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10"); + indexr(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1", "testnull_i", null); + indexr(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2", "testnull_i", "2"); + indexr(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "testnull_i", null); + indexr(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "testnull_i", "4"); + indexr(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5", "testnull_i", null); + indexr(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6", "testnull_i", "6"); + indexr(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7", "testnull_i", null); + indexr(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8", "testnull_i", "8"); + indexr(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9", "testnull_i", null); + indexr(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10", "testnull_i", "10"); commit(); @@ -355,11 +355,28 @@ public class JdbcTest extends AbstractFullDistribZkTestBase { private void testDriverMetadata() throws Exception { String collection = DEFAULT_COLLECTION; - String connectionString = "jdbc:solr://" + zkServer.getZkAddress() + "?collection=" + collection + - "&username=&password=&testKey1=testValue&testKey2"; - String sql = "select id, a_i, a_s, a_f as my_float_col from " + collection + " order by a_i desc limit 2"; - try (Connection con = DriverManager.getConnection(connectionString)) { + String connectionString1 = "jdbc:solr://" + zkServer.getZkAddress() + "?collection=" + collection + + "&username=&password=&testKey1=testValue&testKey2"; + Properties properties1 = new Properties(); + + String sql = "select id, a_i, a_s, a_f as my_float_col, testnull_i from " + collection + + " order by a_i desc"; + + String connectionString2 = "jdbc:solr://" + zkServer.getZkAddress() + "?collection=" + collection + + "&aggregationMode=map_reduce&numWorkers=2&username=&password=&testKey1=testValue&testKey2"; + Properties properties2 = new Properties(); + + String sql2 = sql + " limit 2"; + + //testJDBCMethods(collection, connectionString1, properties1, sql); + //testJDBCMethods(collection, connectionString2, properties2, sql); + testJDBCMethods(collection, connectionString1, properties1, sql2); + testJDBCMethods(collection, connectionString2, properties2, sql2); + } + + private void testJDBCMethods(String collection, String connectionString, Properties properties, String sql) throws Exception { + try (Connection con = DriverManager.getConnection(connectionString, properties)) { assertTrue(con.isValid(DEFAULT_CONNECTION_TIMEOUT)); assertEquals(collection, con.getCatalog()); @@ -407,32 +424,37 @@ public class JdbcTest extends AbstractFullDistribZkTestBase { assertNotNull(resultSetMetaData); - assertEquals(4, resultSetMetaData.getColumnCount()); + assertEquals(5, resultSetMetaData.getColumnCount()); assertEquals("id", resultSetMetaData.getColumnName(1)); assertEquals("a_i", resultSetMetaData.getColumnName(2)); assertEquals("a_s", resultSetMetaData.getColumnName(3)); assertEquals("a_f", resultSetMetaData.getColumnName(4)); + assertEquals("testnull_i", resultSetMetaData.getColumnName(5)); assertEquals("id", resultSetMetaData.getColumnLabel(1)); assertEquals("a_i", resultSetMetaData.getColumnLabel(2)); assertEquals("a_s", resultSetMetaData.getColumnLabel(3)); assertEquals("my_float_col", resultSetMetaData.getColumnLabel(4)); + assertEquals("testnull_i", resultSetMetaData.getColumnLabel(5)); assertEquals("id".length(), resultSetMetaData.getColumnDisplaySize(1)); assertEquals("a_i".length(), resultSetMetaData.getColumnDisplaySize(2)); assertEquals("a_s".length(), resultSetMetaData.getColumnDisplaySize(3)); assertEquals("my_float_col".length(), resultSetMetaData.getColumnDisplaySize(4)); + assertEquals("testnull_i".length(), resultSetMetaData.getColumnDisplaySize(5)); assertEquals("Long", resultSetMetaData.getColumnTypeName(1)); assertEquals("Long", resultSetMetaData.getColumnTypeName(2)); assertEquals("String", resultSetMetaData.getColumnTypeName(3)); assertEquals("Double", resultSetMetaData.getColumnTypeName(4)); + assertEquals("Long", resultSetMetaData.getColumnTypeName(5)); assertEquals(Types.DOUBLE, resultSetMetaData.getColumnType(1)); assertEquals(Types.DOUBLE, resultSetMetaData.getColumnType(2)); assertEquals(Types.VARCHAR, resultSetMetaData.getColumnType(3)); assertEquals(Types.DOUBLE, resultSetMetaData.getColumnType(4)); + assertEquals(Types.DOUBLE, resultSetMetaData.getColumnType(5)); } private void checkResultSet(ResultSet rs) throws Exception { @@ -443,72 +465,199 @@ public class JdbcTest extends AbstractFullDistribZkTestBase { assertTrue(rs.next()); assertEquals(14L, rs.getObject("a_i")); + assertFalse(rs.wasNull()); assertEquals(14L, rs.getObject(2)); + assertFalse(rs.wasNull()); assertEquals(14L, rs.getLong("a_i")); + assertFalse(rs.wasNull()); assertEquals(14L, rs.getLong(2)); + assertFalse(rs.wasNull()); assertEquals(14D, rs.getDouble("a_i"), 0); + assertFalse(rs.wasNull()); assertEquals(14D, rs.getDouble(2), 0); + assertFalse(rs.wasNull()); assertEquals(14f, rs.getFloat("a_i"), 0); + assertFalse(rs.wasNull()); assertEquals(14f, rs.getFloat(2), 0); + assertFalse(rs.wasNull()); assertEquals(14, rs.getShort("a_i")); + assertFalse(rs.wasNull()); assertEquals(14, rs.getShort(2)); + assertFalse(rs.wasNull()); assertEquals(14, rs.getByte("a_i")); + assertFalse(rs.wasNull()); assertEquals(14, rs.getByte(2)); + assertFalse(rs.wasNull()); assertEquals("hello0", rs.getObject("a_s")); + assertFalse(rs.wasNull()); assertEquals("hello0", rs.getObject(3)); + assertFalse(rs.wasNull()); assertEquals("hello0", rs.getString("a_s")); + assertFalse(rs.wasNull()); assertEquals("hello0", rs.getString(3)); + assertFalse(rs.wasNull()); assertEquals(10D, rs.getObject("my_float_col")); + assertFalse(rs.wasNull()); assertEquals(10D, rs.getObject(4)); + assertFalse(rs.wasNull()); assertEquals(10D, rs.getDouble("my_float_col"), 0); + assertFalse(rs.wasNull()); assertEquals(10D, rs.getDouble(4), 0); + assertFalse(rs.wasNull()); assertEquals(10F, rs.getFloat("my_float_col"), 0); + assertFalse(rs.wasNull()); assertEquals(10F, rs.getFloat(4), 0); + assertFalse(rs.wasNull()); assertEquals(10, rs.getInt("my_float_col"), 0); + assertFalse(rs.wasNull()); assertEquals(10, rs.getInt(4), 0); + assertFalse(rs.wasNull()); assertEquals(10L, rs.getLong("my_float_col"), 0); + assertFalse(rs.wasNull()); assertEquals(10L, rs.getLong(4), 0); + assertFalse(rs.wasNull()); assertEquals(10, rs.getShort("my_float_col"), 0); + assertFalse(rs.wasNull()); assertEquals(10, rs.getShort(4), 0); + assertFalse(rs.wasNull()); assertEquals(10, rs.getByte("my_float_col"), 0); + assertFalse(rs.wasNull()); assertEquals(10, rs.getByte(4), 0); + assertFalse(rs.wasNull()); + + assertEquals(10L, rs.getObject("testnull_i")); + assertFalse(rs.wasNull()); + assertEquals(10L, rs.getObject(5)); + assertFalse(rs.wasNull()); + assertEquals("10", rs.getString("testnull_i")); + assertFalse(rs.wasNull()); + assertEquals("10", rs.getString(5)); + assertFalse(rs.wasNull()); + assertEquals(10D, rs.getDouble("testnull_i"), 0); + assertFalse(rs.wasNull()); + assertEquals(10D, rs.getDouble(5), 0); + assertFalse(rs.wasNull()); + assertEquals(10F, rs.getFloat("testnull_i"), 0); + assertFalse(rs.wasNull()); + assertEquals(10F, rs.getFloat(5), 0); + assertFalse(rs.wasNull()); + assertEquals(10, rs.getInt("testnull_i"), 0); + assertFalse(rs.wasNull()); + assertEquals(10, rs.getInt(5), 0); + assertFalse(rs.wasNull()); + assertEquals(10L, rs.getLong("testnull_i"), 0); + assertFalse(rs.wasNull()); + assertEquals(10L, rs.getLong(5), 0); + assertFalse(rs.wasNull()); + assertEquals(10, rs.getShort("testnull_i"), 0); + assertFalse(rs.wasNull()); + assertEquals(10, rs.getShort(5), 0); + assertFalse(rs.wasNull()); + assertEquals(10, rs.getByte("testnull_i"), 0); + assertFalse(rs.wasNull()); + assertEquals(10, rs.getByte(5), 0); + assertFalse(rs.wasNull()); + assertTrue(rs.next()); assertEquals(13L, rs.getObject("a_i")); + assertFalse(rs.wasNull()); assertEquals(13L, rs.getObject(2)); + assertFalse(rs.wasNull()); assertEquals(13L, rs.getLong("a_i")); + assertFalse(rs.wasNull()); assertEquals(13L, rs.getLong(2)); + assertFalse(rs.wasNull()); assertEquals(13D, rs.getDouble("a_i"), 0); + assertFalse(rs.wasNull()); assertEquals(13D, rs.getDouble(2), 0); + assertFalse(rs.wasNull()); assertEquals(13f, rs.getFloat("a_i"), 0); + assertFalse(rs.wasNull()); assertEquals(13f, rs.getFloat(2), 0); + assertFalse(rs.wasNull()); assertEquals(13, rs.getShort("a_i")); + assertFalse(rs.wasNull()); assertEquals(13, rs.getShort(2)); + assertFalse(rs.wasNull()); assertEquals(13, rs.getByte("a_i")); + assertFalse(rs.wasNull()); assertEquals(13, rs.getByte(2)); + assertFalse(rs.wasNull()); assertEquals("hello3", rs.getObject("a_s")); + assertFalse(rs.wasNull()); assertEquals("hello3", rs.getObject(3)); + assertFalse(rs.wasNull()); assertEquals("hello3", rs.getString("a_s")); + assertFalse(rs.wasNull()); assertEquals("hello3", rs.getString(3)); + assertFalse(rs.wasNull()); assertEquals(9D, rs.getObject("my_float_col")); + assertFalse(rs.wasNull()); assertEquals(9D, rs.getObject(4)); + assertFalse(rs.wasNull()); assertEquals(9D, rs.getDouble("my_float_col"), 0); + assertFalse(rs.wasNull()); assertEquals(9D, rs.getDouble(4), 0); + assertFalse(rs.wasNull()); assertEquals(9F, rs.getFloat("my_float_col"), 0); + assertFalse(rs.wasNull()); assertEquals(9F, rs.getFloat(4), 0); + assertFalse(rs.wasNull()); assertEquals(9, rs.getInt("my_float_col"), 0); + assertFalse(rs.wasNull()); assertEquals(9, rs.getInt(4), 0); + assertFalse(rs.wasNull()); assertEquals(9L, rs.getLong("my_float_col"), 0); + assertFalse(rs.wasNull()); assertEquals(9L, rs.getLong(4), 0); + assertFalse(rs.wasNull()); assertEquals(9, rs.getShort("my_float_col"), 0); + assertFalse(rs.wasNull()); assertEquals(9, rs.getShort(4), 0); + assertFalse(rs.wasNull()); assertEquals(9, rs.getByte("my_float_col"), 0); + assertFalse(rs.wasNull()); assertEquals(9, rs.getByte(4), 0); + assertFalse(rs.wasNull()); + + assertEquals(null, rs.getObject("testnull_i")); + assertTrue(rs.wasNull()); + assertEquals(null, rs.getObject(5)); + assertTrue(rs.wasNull()); + assertEquals(null, rs.getString("testnull_i")); + assertTrue(rs.wasNull()); + assertEquals(null, rs.getString(5)); + assertTrue(rs.wasNull()); + assertEquals(0D, rs.getDouble("testnull_i"), 0); + assertTrue(rs.wasNull()); + assertEquals(0D, rs.getDouble(5), 0); + assertTrue(rs.wasNull()); + assertEquals(0F, rs.getFloat("testnull_i"), 0); + assertTrue(rs.wasNull()); + assertEquals(0F, rs.getFloat(5), 0); + assertTrue(rs.wasNull()); + assertEquals(0, rs.getInt("testnull_i")); + assertTrue(rs.wasNull()); + assertEquals(0, rs.getInt(5)); + assertTrue(rs.wasNull()); + assertEquals(0L, rs.getLong("testnull_i")); + assertTrue(rs.wasNull()); + assertEquals(0L, rs.getLong(5)); + assertTrue(rs.wasNull()); + assertEquals(0, rs.getShort("testnull_i")); + assertTrue(rs.wasNull()); + assertEquals(0, rs.getShort(5)); + assertTrue(rs.wasNull()); + assertEquals(0, rs.getByte("testnull_i")); + assertTrue(rs.wasNull()); + assertEquals(0, rs.getByte(5)); + assertTrue(rs.wasNull()); assertFalse(rs.next()); } From 47fb35c20ac22c0226daa583e0179f8805e54664 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 28 Jan 2016 11:29:37 -0500 Subject: [PATCH 04/21] fix false failures from test bugs --- .../index/TestAllFilesCheckIndexHeader.java | 23 +++++++++++++++---- .../index/TestAllFilesDetectTruncation.java | 4 +++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java index 9dffaa8df6c..3d8aeffe3b9 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java @@ -19,6 +19,7 @@ package org.apache.lucene.index; import java.io.EOFException; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import org.apache.lucene.analysis.MockAnalyzer; @@ -80,7 +81,9 @@ public class TestAllFilesCheckIndexHeader extends LuceneTestCase { private void checkIndexHeader(Directory dir) throws IOException { for(String name : dir.listAll()) { - checkOneFile(dir, name); + if (name.equals(IndexWriter.WRITE_LOCK_NAME) == false) { + checkOneFile(dir, name); + } } } @@ -99,15 +102,25 @@ public class TestAllFilesCheckIndexHeader extends LuceneTestCase { if (name.equals(victim) == false) { dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT); } else { - try(IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT); - IndexInput in = dir.openInput(name, IOContext.DEFAULT)) { + + // Iterate until our randomly generated bytes are indeed different from the first bytes of the file ... the vast majority of the + // time this will only require one iteration! + while (true) { + try(IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT); + IndexInput in = dir.openInput(name, IOContext.DEFAULT)) { // keeps same file length, but replaces the first wrongBytes with random bytes: byte[] bytes = new byte[wrongBytes]; random().nextBytes(bytes); out.writeBytes(bytes, 0, bytes.length); - in.seek(wrongBytes); - out.copyBytes(in, victimLength - wrongBytes); + byte[] bytes2 = new byte[wrongBytes]; + in.readBytes(bytes2, 0, bytes2.length); + if (Arrays.equals(bytes, bytes2) == false) { + // We successfully randomly generated bytes that differ from the bytes in the file: + out.copyBytes(in, victimLength - wrongBytes); + break; + } } + } } dirCopy.sync(Collections.singleton(name)); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java index bda5857a7a7..e5aa6c7f9ad 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java @@ -80,7 +80,9 @@ public class TestAllFilesDetectTruncation extends LuceneTestCase { private void checkTruncation(Directory dir) throws IOException { for(String name : dir.listAll()) { - truncateOneFile(dir, name); + if (name.equals(IndexWriter.WRITE_LOCK_NAME) == false) { + truncateOneFile(dir, name); + } } } From 5a4bb9f2a1be048e3ec66d6165a19eb90be30ea9 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 28 Jan 2016 12:17:48 -0500 Subject: [PATCH 05/21] disable ExtraFS for these tests --- .../org/apache/lucene/index/TestAllFilesCheckIndexHeader.java | 2 ++ .../org/apache/lucene/index/TestAllFilesDetectTruncation.java | 2 ++ .../src/test/org/apache/lucene/index/TestSwappedIndexFiles.java | 2 ++ 3 files changed, 6 insertions(+) diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java index 3d8aeffe3b9..30315538ee8 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesCheckIndexHeader.java @@ -30,12 +30,14 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; +import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; /** * Test that a plain default detects broken index headers early (on opening a reader). */ +@SuppressFileSystems("ExtrasFS") public class TestAllFilesCheckIndexHeader extends LuceneTestCase { public void test() throws Exception { Directory dir = newDirectory(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java index e5aa6c7f9ad..347955d639c 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesDetectTruncation.java @@ -30,12 +30,14 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; +import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; /** * Test that a plain default detects index file truncation early (on opening a reader). */ +@SuppressFileSystems("ExtrasFS") public class TestAllFilesDetectTruncation extends LuceneTestCase { public void test() throws Exception { Directory dir = newDirectory(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java index 008e9a69e1f..a9a8c1436d8 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestSwappedIndexFiles.java @@ -29,12 +29,14 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LineFileDocs; +import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; /** * Test that the same file name, but from a different index, is detected as foreign. */ +@SuppressFileSystems("ExtrasFS") public class TestSwappedIndexFiles extends LuceneTestCase { public void test() throws Exception { Directory dir1 = newDirectory(); From 62c9b6a1724fb466fd767457b7bbdeb24e164036 Mon Sep 17 00:00:00 2001 From: Mark Miller Date: Thu, 28 Jan 2016 12:28:10 -0500 Subject: [PATCH 06/21] SOLR-8451: fix randomization in test. --- .../test/org/apache/solr/client/solrj/ConnectionReuseTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/client/solrj/ConnectionReuseTest.java b/solr/core/src/test/org/apache/solr/client/solrj/ConnectionReuseTest.java index 8dbf99c1dda..352010c6b13 100644 --- a/solr/core/src/test/org/apache/solr/client/solrj/ConnectionReuseTest.java +++ b/solr/core/src/test/org/apache/solr/client/solrj/ConnectionReuseTest.java @@ -79,7 +79,7 @@ public class ConnectionReuseTest extends AbstractFullDistribZkTestBase { SolrClient client; HttpClient httpClient = HttpClientUtil.createClient(null); - int rndClient = 0;//random().nextInt(3); + int rndClient = random().nextInt(3); if (rndClient == 0) { client = new ConcurrentUpdateSolrClient(url.toString(), httpClient, 6, 1); // currently only testing with 1 thread } else if (rndClient == 1) { From 9754da625c85b97e21a2bed2dc26246f7a4c1e70 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Thu, 28 Jan 2016 17:39:52 +0000 Subject: [PATCH 07/21] SOLR-8597: add default, no-op QParserPlugin.init(NamedList) method --- .../src/java/org/apache/solr/search/BoostQParserPlugin.java | 4 ---- .../java/org/apache/solr/search/CollapsingQParserPlugin.java | 4 ---- .../org/apache/solr/search/ComplexPhraseQParserPlugin.java | 1 + .../src/java/org/apache/solr/search/DisMaxQParserPlugin.java | 4 ---- .../src/java/org/apache/solr/search/ExportQParserPlugin.java | 3 --- .../org/apache/solr/search/ExtendedDismaxQParserPlugin.java | 4 ---- .../src/java/org/apache/solr/search/FieldQParserPlugin.java | 4 ---- .../java/org/apache/solr/search/FunctionQParserPlugin.java | 4 ---- .../org/apache/solr/search/FunctionRangeQParserPlugin.java | 4 ---- .../src/java/org/apache/solr/search/HashQParserPlugin.java | 4 ---- .../src/java/org/apache/solr/search/JoinQParserPlugin.java | 4 ---- .../src/java/org/apache/solr/search/LuceneQParserPlugin.java | 4 ---- .../src/java/org/apache/solr/search/NestedQParserPlugin.java | 4 ---- .../java/org/apache/solr/search/OldLuceneQParserPlugin.java | 4 ---- .../src/java/org/apache/solr/search/PrefixQParserPlugin.java | 4 ---- solr/core/src/java/org/apache/solr/search/QParserPlugin.java | 4 ++++ .../src/java/org/apache/solr/search/RawQParserPlugin.java | 4 ---- .../src/java/org/apache/solr/search/ReRankQParserPlugin.java | 3 --- .../src/java/org/apache/solr/search/SimpleQParserPlugin.java | 5 ----- .../java/org/apache/solr/search/SpatialBoxQParserPlugin.java | 5 ----- .../org/apache/solr/search/SpatialFilterQParserPlugin.java | 5 ----- .../java/org/apache/solr/search/SurroundQParserPlugin.java | 4 ---- .../src/java/org/apache/solr/search/SwitchQParserPlugin.java | 4 ---- .../src/java/org/apache/solr/search/TermQParserPlugin.java | 4 ---- .../src/java/org/apache/solr/search/TermsQParserPlugin.java | 4 ---- .../src/java/org/apache/solr/search/XmlQParserPlugin.java | 4 ---- .../solr/search/join/BlockJoinParentQParserPlugin.java | 4 ---- .../java/org/apache/solr/search/join/GraphQParserPlugin.java | 4 ---- .../org/apache/solr/search/join/ScoreJoinQParserPlugin.java | 4 ---- .../java/org/apache/solr/search/mlt/MLTQParserPlugin.java | 5 ----- .../src/test/org/apache/solr/search/FooQParserPlugin.java | 4 ---- .../org/apache/solr/search/TestAnalyticsQParserPlugin.java | 4 ---- .../src/test/org/apache/solr/search/TestRankQueryPlugin.java | 4 ---- 33 files changed, 5 insertions(+), 126 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/BoostQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/BoostQParserPlugin.java index d73b02544a5..5f7add8b276 100644 --- a/solr/core/src/java/org/apache/solr/search/BoostQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/BoostQParserPlugin.java @@ -40,10 +40,6 @@ public class BoostQParserPlugin extends QParserPlugin { public static final String NAME = "boost"; public static String BOOSTFUNC = "b"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index a74e3f2be4b..cc7e6dad540 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -122,10 +122,6 @@ public class CollapsingQParserPlugin extends QParserPlugin { public static final String HINT_MULTI_DOCVALUES = "multi_docvalues"; - public void init(NamedList namedList) { - - } - public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest request) { return new CollapsingQParser(qstr, localParams, params, request); } diff --git a/solr/core/src/java/org/apache/solr/search/ComplexPhraseQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/ComplexPhraseQParserPlugin.java index 1452eeef5e3..51d819efdea 100644 --- a/solr/core/src/java/org/apache/solr/search/ComplexPhraseQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/ComplexPhraseQParserPlugin.java @@ -40,6 +40,7 @@ public class ComplexPhraseQParserPlugin extends QParserPlugin { @Override public void init(NamedList args) { + super.init(args); if (args != null) { Object val = args.get("inOrder"); if (val != null) { diff --git a/solr/core/src/java/org/apache/solr/search/DisMaxQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/DisMaxQParserPlugin.java index d2efbcc3895..f8bfecf2f94 100644 --- a/solr/core/src/java/org/apache/solr/search/DisMaxQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/DisMaxQParserPlugin.java @@ -112,10 +112,6 @@ import org.apache.solr.request.SolrQueryRequest; public class DisMaxQParserPlugin extends QParserPlugin { public static final String NAME = "dismax"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new DisMaxQParser(qstr, localParams, params, req); diff --git a/solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java index f98e479c440..4ed3e3bd05d 100644 --- a/solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java @@ -34,9 +34,6 @@ public class ExportQParserPlugin extends QParserPlugin { public static final String NAME = "xport"; - public void init(NamedList namedList) { - } - public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest request) { return new ExportQParser(qstr, localParams, params, request); } diff --git a/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java index d7337138752..41121bc4022 100644 --- a/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java @@ -28,10 +28,6 @@ import org.apache.solr.request.SolrQueryRequest; public class ExtendedDismaxQParserPlugin extends QParserPlugin { public static final String NAME = "edismax"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new ExtendedDismaxQParser(qstr, localParams, params, req); diff --git a/solr/core/src/java/org/apache/solr/search/FieldQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/FieldQParserPlugin.java index 4a5199399ad..151647de11f 100644 --- a/solr/core/src/java/org/apache/solr/search/FieldQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/FieldQParserPlugin.java @@ -34,10 +34,6 @@ import org.apache.solr.schema.SchemaField; public class FieldQParserPlugin extends QParserPlugin { public static final String NAME = "field"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { diff --git a/solr/core/src/java/org/apache/solr/search/FunctionQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/FunctionQParserPlugin.java index 00eba696d56..561c5324e79 100644 --- a/solr/core/src/java/org/apache/solr/search/FunctionQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/FunctionQParserPlugin.java @@ -28,10 +28,6 @@ import org.apache.solr.request.SolrQueryRequest; public class FunctionQParserPlugin extends QParserPlugin { public static final String NAME = "func"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new FunctionQParser(qstr, localParams, params, req); diff --git a/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java index 0c3f8c8610f..8db8bbc1325 100644 --- a/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java @@ -38,10 +38,6 @@ import org.apache.solr.search.function.*; public class FunctionRangeQParserPlugin extends QParserPlugin { public static final String NAME = "frange"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { diff --git a/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java index 3b67a96970a..4f5c3a55bed 100644 --- a/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java @@ -56,10 +56,6 @@ public class HashQParserPlugin extends QParserPlugin { public static final String NAME = "hash"; - public void init(NamedList params) { - - } - public QParser createParser(String query, SolrParams localParams, SolrParams params, SolrQueryRequest request) { return new HashQParser(query, localParams, params, request); } diff --git a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java index bdcfd6f663c..af333305e5d 100644 --- a/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -60,10 +60,6 @@ import org.apache.solr.util.RefCounted; public class JoinQParserPlugin extends QParserPlugin { public static final String NAME = "join"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { diff --git a/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java index f2b600c1af1..e92ce318f34 100644 --- a/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/LuceneQParserPlugin.java @@ -36,10 +36,6 @@ import java.util.List; public class LuceneQParserPlugin extends QParserPlugin { public static final String NAME = "lucene"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new LuceneQParser(qstr, localParams, params, req); diff --git a/solr/core/src/java/org/apache/solr/search/NestedQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/NestedQParserPlugin.java index 98b7e9cb4d7..9b0dc5a0797 100644 --- a/solr/core/src/java/org/apache/solr/search/NestedQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/NestedQParserPlugin.java @@ -34,10 +34,6 @@ import org.apache.solr.request.SolrQueryRequest; public class NestedQParserPlugin extends QParserPlugin { public static final String NAME = "query"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { diff --git a/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java index 006e36cab0b..a59f28739c3 100644 --- a/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/OldLuceneQParserPlugin.java @@ -28,10 +28,6 @@ import org.apache.solr.request.SolrQueryRequest; public class OldLuceneQParserPlugin extends QParserPlugin { public static final String NAME = "lucenePlusSort"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new OldLuceneQParser(qstr, localParams, params, req); diff --git a/solr/core/src/java/org/apache/solr/search/PrefixQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/PrefixQParserPlugin.java index e66d4dc4a7a..8aa74461d36 100644 --- a/solr/core/src/java/org/apache/solr/search/PrefixQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/PrefixQParserPlugin.java @@ -34,10 +34,6 @@ import org.apache.solr.schema.SchemaField; public class PrefixQParserPlugin extends QParserPlugin { public static final String NAME = "prefix"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java index 3a38c4cd91e..cf556ff96fe 100644 --- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java @@ -82,6 +82,10 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI /** return a {@link QParser} */ public abstract QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req); + @Override + public void init( NamedList args ) { + } + @Override public String getName() { // TODO: ideally use the NAME property that each qparser plugin has diff --git a/solr/core/src/java/org/apache/solr/search/RawQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/RawQParserPlugin.java index 77fe52393f9..5d93fb94425 100644 --- a/solr/core/src/java/org/apache/solr/search/RawQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/RawQParserPlugin.java @@ -36,10 +36,6 @@ import org.apache.solr.request.SolrQueryRequest; public class RawQParserPlugin extends QParserPlugin { public static final String NAME = "raw"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { diff --git a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java index 046d28b7d36..2b23d53aff6 100644 --- a/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java @@ -64,9 +64,6 @@ public class ReRankQParserPlugin extends QParserPlugin { public static final String NAME = "rerank"; private static Query defaultQuery = new MatchAllDocsQuery(); - public void init(NamedList args) { - } - public QParser createParser(String query, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new ReRankQParser(query, localParams, params, req); } diff --git a/solr/core/src/java/org/apache/solr/search/SimpleQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/SimpleQParserPlugin.java index 2797189e2d7..2cab3c9c4c0 100644 --- a/solr/core/src/java/org/apache/solr/search/SimpleQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/SimpleQParserPlugin.java @@ -91,11 +91,6 @@ public class SimpleQParserPlugin extends QParserPlugin { OPERATORS.put(SimpleParams.NEAR_OPERATOR, SimpleQueryParser.NEAR_OPERATOR); } - /** No initialization is necessary so this method is empty. */ - @Override - public void init(NamedList args) { - } - /** Returns a QParser that will create a query by using Lucene's SimpleQueryParser. */ @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { diff --git a/solr/core/src/java/org/apache/solr/search/SpatialBoxQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/SpatialBoxQParserPlugin.java index 00afb1a1c53..0f2bccbf524 100644 --- a/solr/core/src/java/org/apache/solr/search/SpatialBoxQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/SpatialBoxQParserPlugin.java @@ -31,9 +31,4 @@ public class SpatialBoxQParserPlugin extends SpatialFilterQParserPlugin { return new SpatialFilterQParser(qstr, localParams, params, req, true); } - @Override - public void init(NamedList args) { - - } - } diff --git a/solr/core/src/java/org/apache/solr/search/SpatialFilterQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/SpatialFilterQParserPlugin.java index c8010a47287..7269bb5feff 100644 --- a/solr/core/src/java/org/apache/solr/search/SpatialFilterQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/SpatialFilterQParserPlugin.java @@ -56,10 +56,5 @@ public class SpatialFilterQParserPlugin extends QParserPlugin { return new SpatialFilterQParser(qstr, localParams, params, req, false); } - @Override - public void init(NamedList args) { - - } - } diff --git a/solr/core/src/java/org/apache/solr/search/SurroundQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/SurroundQParserPlugin.java index c336a373944..c87c3f046d9 100644 --- a/solr/core/src/java/org/apache/solr/search/SurroundQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/SurroundQParserPlugin.java @@ -48,10 +48,6 @@ public class SurroundQParserPlugin extends QParserPlugin { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String NAME = "surround"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { diff --git a/solr/core/src/java/org/apache/solr/search/SwitchQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/SwitchQParserPlugin.java index 8f3944849cf..d3947a91743 100644 --- a/solr/core/src/java/org/apache/solr/search/SwitchQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/SwitchQParserPlugin.java @@ -151,10 +151,6 @@ public class SwitchQParserPlugin extends QParserPlugin { */ public static String SWITCH_DEFAULT = "default"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { diff --git a/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java index 27dd8c38b51..2520bf54771 100644 --- a/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/TermQParserPlugin.java @@ -43,10 +43,6 @@ import org.apache.solr.schema.FieldType; public class TermQParserPlugin extends QParserPlugin { public static final String NAME = "term"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new QParser(qstr, localParams, params, req) { diff --git a/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java index 434b1e341d4..0233bf0bbb2 100644 --- a/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java @@ -58,10 +58,6 @@ public class TermsQParserPlugin extends QParserPlugin { /** Choose the internal algorithm */ private static final String METHOD = "method"; - @Override - public void init(NamedList args) { - } - private static enum Method { termsFilter { @Override diff --git a/solr/core/src/java/org/apache/solr/search/XmlQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/XmlQParserPlugin.java index bb33e0b398b..71f1fb8a25a 100755 --- a/solr/core/src/java/org/apache/solr/search/XmlQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/XmlQParserPlugin.java @@ -58,10 +58,6 @@ public class XmlQParserPlugin extends QParserPlugin { } - @Override - public void init(NamedList args) { - } - public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new XmlQParser(qstr, localParams, params, req); diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParserPlugin.java index 22d9fffe36b..70611911e66 100644 --- a/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParserPlugin.java @@ -41,9 +41,5 @@ public class BlockJoinParentQParserPlugin extends QParserPlugin { protected QParser createBJQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new BlockJoinParentQParser(qstr, localParams, params, req); } - - @Override - public void init(NamedList args) { - } } diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/GraphQParserPlugin.java index 6bda867ee82..c60e54aeaa1 100644 --- a/solr/core/src/java/org/apache/solr/search/join/GraphQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/join/GraphQParserPlugin.java @@ -31,10 +31,6 @@ public class GraphQParserPlugin extends QParserPlugin { // Graph Query Parser parser name public static final String NAME = "graph"; - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { // return the graph query parser for this request. diff --git a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java index ebd6155436b..e27e22794cd 100644 --- a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java @@ -206,10 +206,6 @@ public class ScoreJoinQParserPlugin extends QParserPlugin { } } - @Override - public void init(NamedList args) { - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { diff --git a/solr/core/src/java/org/apache/solr/search/mlt/MLTQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/mlt/MLTQParserPlugin.java index 1a2f36961f6..c18e546298f 100644 --- a/solr/core/src/java/org/apache/solr/search/mlt/MLTQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/mlt/MLTQParserPlugin.java @@ -28,11 +28,6 @@ import org.apache.solr.search.QParserPlugin; public class MLTQParserPlugin extends QParserPlugin { public static final String NAME = "mlt"; - @Override - public void init(NamedList args) { - - } - @Override public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { diff --git a/solr/core/src/test/org/apache/solr/search/FooQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/FooQParserPlugin.java index 9d406ad6cc5..2df670540b8 100644 --- a/solr/core/src/test/org/apache/solr/search/FooQParserPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/FooQParserPlugin.java @@ -30,10 +30,6 @@ public class FooQParserPlugin extends QParserPlugin { public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new FooQParser(qstr, localParams, params, req); } - - @Override - public void init(NamedList args) { - } } class FooQParser extends QParser { diff --git a/solr/core/src/test/org/apache/solr/search/TestAnalyticsQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestAnalyticsQParserPlugin.java index c2ea050e627..1a86f482a0a 100644 --- a/solr/core/src/test/org/apache/solr/search/TestAnalyticsQParserPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/TestAnalyticsQParserPlugin.java @@ -41,10 +41,6 @@ import java.io.IOException; public class TestAnalyticsQParserPlugin extends QParserPlugin { - public void init(NamedList params) { - - } - public QParser createParser(String query, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new TestAnalyticsQueryParser(query, localParams, params, req); } diff --git a/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java b/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java index b9ec6998def..dd42b577aa5 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java @@ -72,10 +72,6 @@ import org.junit.Ignore; public class TestRankQueryPlugin extends QParserPlugin { - public void init(NamedList params) { - - } - public QParser createParser(String query, SolrParams localParams, SolrParams params, SolrQueryRequest req) { return new TestRankQueryParser(query, localParams, params, req); } From 70b1eb03c4463c09523297bb046b239e7fa3c2c4 Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Thu, 28 Jan 2016 17:41:34 +0000 Subject: [PATCH 08/21] SOLR-8597: add default, no-op QParserPlugin.init(NamedList) method (solr/CHANGES.txt) --- solr/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 584b8271ae9..5ff042f1542 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -576,6 +576,8 @@ Other Changes * SOLR-8595: Use BinaryRequestWriter by default in HttpSolrClient and ConcurrentUpdateSolrClient. (shalin) +* SOLR-8597: add default, no-op QParserPlugin.init(NamedList) method (Christine Poerschke) + ================== 5.4.1 ================== Bug Fixes From da5cc3cad56c97d203d9f1ffe267e24ce615766b Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 28 Jan 2016 16:24:27 -0500 Subject: [PATCH 09/21] add CrankyPointFormat --- .../lucene/index/SegmentCoreReaders.java | 4 +- .../lucene/codecs/cranky/CrankyCodec.java | 6 + .../codecs/cranky/CrankyPointFormat.java | 176 ++++++++++++++++++ 3 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java b/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java index 414c70732f1..b5d12219914 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java @@ -166,7 +166,7 @@ final class SegmentCoreReaders { } } - private void notifyCoreClosedListeners(Throwable th) { + private void notifyCoreClosedListeners(Throwable th) throws IOException { synchronized(coreClosedListeners) { for (CoreClosedListener listener : coreClosedListeners) { // SegmentReader uses our instance as its @@ -181,7 +181,7 @@ final class SegmentCoreReaders { } } } - IOUtils.reThrowUnchecked(th); + IOUtils.reThrow(th); } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java index 7925356f5dd..3367d429093 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCodec.java @@ -26,6 +26,7 @@ import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.LiveDocsFormat; import org.apache.lucene.codecs.NormsFormat; +import org.apache.lucene.codecs.PointFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; import org.apache.lucene.codecs.StoredFieldsFormat; @@ -91,6 +92,11 @@ public class CrankyCodec extends FilterCodec { return new CrankyCompoundFormat(delegate.compoundFormat(), random); } + @Override + public PointFormat pointFormat() { + return new CrankyPointFormat(delegate.pointFormat(), random); + } + @Override public String toString() { return "Cranky(" + delegate + ")"; diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java new file mode 100644 index 00000000000..ed2b7a3da91 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPointFormat.java @@ -0,0 +1,176 @@ +package org.apache.lucene.codecs.cranky; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Random; + +import org.apache.lucene.codecs.PointFormat; +import org.apache.lucene.codecs.PointReader; +import org.apache.lucene.codecs.PointWriter; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +class CrankyPointFormat extends PointFormat { + PointFormat delegate; + Random random; + + CrankyPointFormat(PointFormat delegate, Random random) { + this.delegate = delegate; + this.random = random; + } + + @Override + public PointWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new CrankyPointWriter(delegate.fieldsWriter(state), random); + } + + @Override + public PointReader fieldsReader(SegmentReadState state) throws IOException { + return new CrankyPointReader(delegate.fieldsReader(state), random); + } + + static class CrankyPointWriter extends PointWriter { + final PointWriter delegate; + final Random random; + + public CrankyPointWriter(PointWriter delegate, Random random) { + this.delegate = delegate; + this.random = random; + } + + @Override + public void writeField(FieldInfo fieldInfo, PointReader values) throws IOException { + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + delegate.writeField(fieldInfo, values); + } + + @Override + public void finish() throws IOException { + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + delegate.finish(); + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + } + + @Override + public void merge(MergeState mergeState) throws IOException { + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + delegate.merge(mergeState); + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + } + + @Override + public void close() throws IOException { + delegate.close(); + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + } + } + + static class CrankyPointReader extends PointReader { + final PointReader delegate; + final Random random; + public CrankyPointReader(PointReader delegate, Random random) { + this.delegate = delegate; + this.random = random; + } + + @Override + public void checkIntegrity() throws IOException { + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + delegate.checkIntegrity(); + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + } + + @Override + public void intersect(String fieldName, IntersectVisitor visitor) throws IOException { + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + delegate.intersect(fieldName, visitor); + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + } + + @Override + public byte[] getMinPackedValue(String fieldName) throws IOException { + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + return delegate.getMinPackedValue(fieldName); + } + + @Override + public byte[] getMaxPackedValue(String fieldName) throws IOException { + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + return delegate.getMaxPackedValue(fieldName); + } + + @Override + public int getNumDimensions(String fieldName) throws IOException { + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + return delegate.getNumDimensions(fieldName); + } + + @Override + public int getBytesPerDimension(String fieldName) throws IOException { + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + return delegate.getBytesPerDimension(fieldName); + } + + @Override + public void close() throws IOException { + delegate.close(); + if (random.nextInt(100) == 0) { + throw new IOException("Fake IOException"); + } + } + + @Override + public long ramBytesUsed() { + return delegate.ramBytesUsed(); + } + } +} From 0607f007bfa1dd3f9c081a354cd7cd1031ac1fa3 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Thu, 28 Jan 2016 19:10:09 -0500 Subject: [PATCH 10/21] throw IOExc, not RuntimeException wrapping IOExc, if we hit exc while closing IndexReader --- lucene/core/src/java/org/apache/lucene/index/IndexReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexReader.java b/lucene/core/src/java/org/apache/lucene/index/IndexReader.java index f20b894b362..f9f80f5788b 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexReader.java @@ -138,7 +138,7 @@ public abstract class IndexReader implements Closeable { parentReaders.add(reader); } - private void notifyReaderClosedListeners(Throwable th) { + private void notifyReaderClosedListeners(Throwable th) throws IOException { synchronized(readerClosedListeners) { for(ReaderClosedListener listener : readerClosedListeners) { try { @@ -151,7 +151,7 @@ public abstract class IndexReader implements Closeable { } } } - IOUtils.reThrowUnchecked(th); + IOUtils.reThrow(th); } } From ae6a971c35422b091fab1949dee24941bec64d3f Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 29 Jan 2016 04:27:27 -0500 Subject: [PATCH 11/21] LUCENE-7000: Split BasePointFormatTestCase from TestPointValues --- .../simpletext/TestSimpleTextPointFormat.java | 33 + .../lucene60/TestLucene60PointFormat.java | 83 ++ .../apache/lucene/index/TestPointValues.java | 922 +----------------- .../lucene/index/BasePointFormatTestCase.java | 882 +++++++++++++++++ .../asserting/TestAssertingPointFormat.java | 31 + 5 files changed, 1053 insertions(+), 898 deletions(-) create mode 100644 lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextPointFormat.java create mode 100644 lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointFormat.java create mode 100644 lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java create mode 100644 lucene/test-framework/src/test/org/apache/lucene/codecs/asserting/TestAssertingPointFormat.java diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextPointFormat.java b/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextPointFormat.java new file mode 100644 index 00000000000..6ef9a61dcf7 --- /dev/null +++ b/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextPointFormat.java @@ -0,0 +1,33 @@ +package org.apache.lucene.codecs.simpletext; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.index.BasePointFormatTestCase; + +/** + * Tests SimpleText's point format + */ +public class TestSimpleTextPointFormat extends BasePointFormatTestCase { + private final Codec codec = new SimpleTextCodec(); + + @Override + protected Codec getCodec() { + return codec; + } +} diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointFormat.java new file mode 100644 index 00000000000..c154f53e6c9 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointFormat.java @@ -0,0 +1,83 @@ +package org.apache.lucene.codecs.lucene60; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.PointFormat; +import org.apache.lucene.codecs.PointReader; +import org.apache.lucene.codecs.PointWriter; +import org.apache.lucene.index.BasePointFormatTestCase; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.util.TestUtil; + +/** + * Tests Lucene60PointFormat + */ +public class TestLucene60PointFormat extends BasePointFormatTestCase { + private final Codec codec; + + public TestLucene60PointFormat() { + // standard issue + Codec defaultCodec = TestUtil.getDefaultCodec(); + if (random().nextBoolean()) { + // randomize parameters + int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 500); + double maxMBSortInHeap = 0.1 + (3*random().nextDouble()); + if (VERBOSE) { + System.out.println("TEST: using Lucene60PointFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap); + } + + // sneaky impersonation! + codec = new FilterCodec(defaultCodec.getName(), defaultCodec) { + @Override + public PointFormat pointFormat() { + return new PointFormat() { + @Override + public PointWriter fieldsWriter(SegmentWriteState writeState) throws IOException { + return new Lucene60PointWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap); + } + + @Override + public PointReader fieldsReader(SegmentReadState readState) throws IOException { + return new Lucene60PointReader(readState); + } + }; + } + }; + } else { + // standard issue + codec = defaultCodec; + } + } + + @Override + protected Codec getCodec() { + return codec; + } + + @Override + public void testMergeStability() throws Exception { + assumeFalse("TODO: mess with the parameters and test gets angry!", codec instanceof FilterCodec); + super.testMergeStability(); + } + +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java index f8185673752..6d0a82ffc50 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java @@ -17,451 +17,22 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.io.IOException; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.BitSet; -import java.util.List; - import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.PointFormat; -import org.apache.lucene.codecs.PointReader; -import org.apache.lucene.codecs.PointWriter; -import org.apache.lucene.codecs.FilterCodec; -import org.apache.lucene.codecs.lucene60.Lucene60PointReader; -import org.apache.lucene.codecs.lucene60.Lucene60PointWriter; import org.apache.lucene.document.BinaryPoint; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.PointValues.IntersectVisitor; -import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.MockDirectoryWrapper; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.TestUtil; -// TODO: factor out a BaseTestDimensionFormat - +/** Test Indexing/IndexWriter with points */ public class TestPointValues extends LuceneTestCase { - public void testBasic() throws Exception { - Directory dir = getDirectory(20); - // TODO: randomize codec once others support points format - IndexWriterConfig iwc = newIndexWriterConfig(); - iwc.setMergePolicy(newLogMergePolicy()); - IndexWriter w = new IndexWriter(dir, iwc); - byte[] point = new byte[4]; - for(int i=0;i<20;i++) { - Document doc = new Document(); - NumericUtils.intToBytes(i, point, 0); - doc.add(new BinaryPoint("dim", point)); - w.addDocument(doc); - } - w.forceMerge(1); - w.close(); - - DirectoryReader r = DirectoryReader.open(dir); - LeafReader sub = getOnlySegmentReader(r); - PointValues values = sub.getPointValues(); - - // Simple test: make sure intersect can visit every doc: - BitSet seen = new BitSet(); - values.intersect("dim", - new IntersectVisitor() { - @Override - public Relation compare(byte[] minPacked, byte[] maxPacked) { - return Relation.CELL_CROSSES_QUERY; - } - public void visit(int docID) { - throw new IllegalStateException(); - } - public void visit(int docID, byte[] packedValue) { - seen.set(docID); - assertEquals(docID, NumericUtils.bytesToInt(packedValue, 0)); - } - }); - assertEquals(20, seen.cardinality()); - IOUtils.close(r, dir); - } - - public void testMerge() throws Exception { - Directory dir = getDirectory(20); - IndexWriterConfig iwc = newIndexWriterConfig(); - iwc.setMergePolicy(newLogMergePolicy()); - IndexWriter w = new IndexWriter(dir, iwc); - byte[] point = new byte[4]; - for(int i=0;i<20;i++) { - Document doc = new Document(); - NumericUtils.intToBytes(i, point, 0); - doc.add(new BinaryPoint("dim", point)); - w.addDocument(doc); - if (i == 10) { - w.commit(); - } - } - w.forceMerge(1); - w.close(); - - DirectoryReader r = DirectoryReader.open(dir); - LeafReader sub = getOnlySegmentReader(r); - PointValues values = sub.getPointValues(); - - // Simple test: make sure intersect can visit every doc: - BitSet seen = new BitSet(); - values.intersect("dim", - new IntersectVisitor() { - @Override - public Relation compare(byte[] minPacked, byte[] maxPacked) { - return Relation.CELL_CROSSES_QUERY; - } - public void visit(int docID) { - throw new IllegalStateException(); - } - public void visit(int docID, byte[] packedValue) { - seen.set(docID); - assertEquals(docID, NumericUtils.bytesToInt(packedValue, 0)); - } - }); - assertEquals(20, seen.cardinality()); - IOUtils.close(r, dir); - } - - public void testAllPointDocsDeletedInSegment() throws Exception { - Directory dir = getDirectory(20); - IndexWriterConfig iwc = newIndexWriterConfig(); - IndexWriter w = new IndexWriter(dir, iwc); - byte[] point = new byte[4]; - for(int i=0;i<10;i++) { - Document doc = new Document(); - NumericUtils.intToBytes(i, point, 0); - doc.add(new BinaryPoint("dim", point)); - doc.add(new NumericDocValuesField("id", i)); - doc.add(newStringField("x", "x", Field.Store.NO)); - w.addDocument(doc); - } - w.addDocument(new Document()); - w.deleteDocuments(new Term("x", "x")); - if (random().nextBoolean()) { - w.forceMerge(1); - } - w.close(); - DirectoryReader r = DirectoryReader.open(dir); - assertEquals(1, r.numDocs()); - PointValues values = MultiPointValues.get(r); - Bits liveDocs = MultiFields.getLiveDocs(r); - NumericDocValues idValues = MultiDocValues.getNumericValues(r, "id"); - - if (values != null) { - BitSet seen = new BitSet(); - values.intersect("dim", - new IntersectVisitor() { - @Override - public Relation compare(byte[] minPacked, byte[] maxPacked) { - return Relation.CELL_CROSSES_QUERY; - } - public void visit(int docID) { - throw new IllegalStateException(); - } - public void visit(int docID, byte[] packedValue) { - if (liveDocs.get(docID)) { - seen.set(docID); - } - assertEquals(idValues.get(docID), NumericUtils.bytesToInt(packedValue, 0)); - } - }); - assertEquals(0, seen.cardinality()); - } - IOUtils.close(r, dir); - } - - /** Make sure we close open files, delete temp files, etc., on exception */ - public void testWithExceptions() throws Exception { - int numDocs = atLeast(10000); - int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES); - int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS); - - byte[][][] docValues = new byte[numDocs][][]; - - for(int docID=0;docID docValues = new ArrayList<>(); - List docIDs = new ArrayList<>(); - - for(int docID=0;docID 0) { - docValues[docID][theEqualDim] = docValues[0][theEqualDim]; - } - } - - verify(docValues, null, numDims, numBytesPerDim); - } - - // Tests on N-dimensional points where each dimension is a BigInteger - public void testBigIntNDims() throws Exception { - - int numDocs = atLeast(1000); - try (Directory dir = getDirectory(numDocs)) { - int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES); - int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS); - IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); - // We rely on docIDs not changing: - iwc.setMergePolicy(newLogMergePolicy()); - RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); - BigInteger[][] docs = new BigInteger[numDocs][]; - - for(int docID=0;docID " + values[dim]); - } - } - docs[docID] = values; - Document doc = new Document(); - doc.add(new BinaryPoint("field", bytes)); - w.addDocument(doc); - } - - DirectoryReader r = w.getReader(); - w.close(); - - PointValues dimValues = MultiPointValues.get(r); - - int iters = atLeast(100); - for(int iter=0;iter= 0; - - if (max.compareTo(queryMin[dim]) < 0 || min.compareTo(queryMax[dim]) > 0) { - return Relation.CELL_OUTSIDE_QUERY; - } else if (min.compareTo(queryMin[dim]) < 0 || max.compareTo(queryMax[dim]) > 0) { - crosses = true; - } - } - - if (crosses) { - return Relation.CELL_CROSSES_QUERY; - } else { - return Relation.CELL_INSIDE_QUERY; - } - } - }); - - for(int docID=0;docID 0) { - expected = false; - break; - } - } - boolean actual = hits.get(docID); - assertEquals("docID=" + docID, expected, actual); - } - } - r.close(); - } - } - - public void testRandomBinaryTiny() throws Exception { - doTestRandomBinary(10); - } - - public void testRandomBinaryMedium() throws Exception { - doTestRandomBinary(10000); - } - - @Nightly - public void testRandomBinaryBig() throws Exception { - assumeFalse("too slow with SimpleText", Codec.getDefault().getName().equals("SimpleText")); - doTestRandomBinary(200000); - } // Suddenly add points to an existing field: public void testUpgradeFieldToPoints() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -479,7 +50,7 @@ public class TestPointValues extends LuceneTestCase { // Illegal schema change tests: public void testIllegalDimChangeOneDoc() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -496,7 +67,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalDimChangeTwoDocs() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -515,7 +86,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalDimChangeTwoSegments() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -535,7 +106,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalDimChangeTwoWriters() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -557,7 +128,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalDimChangeViaAddIndexesDirectory() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -565,7 +136,7 @@ public class TestPointValues extends LuceneTestCase { w.addDocument(doc); w.close(); - Directory dir2 = getDirectory(1); + Directory dir2 = newDirectory(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); w = new IndexWriter(dir2, iwc); doc = new Document(); @@ -580,7 +151,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalDimChangeViaAddIndexesCodecReader() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -588,7 +159,7 @@ public class TestPointValues extends LuceneTestCase { w.addDocument(doc); w.close(); - Directory dir2 = getDirectory(1); + Directory dir2 = newDirectory(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); w = new IndexWriter(dir2, iwc); doc = new Document(); @@ -604,7 +175,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalDimChangeViaAddIndexesSlowCodecReader() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -612,7 +183,7 @@ public class TestPointValues extends LuceneTestCase { w.addDocument(doc); w.close(); - Directory dir2 = getDirectory(1); + Directory dir2 = newDirectory(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); w = new IndexWriter(dir2, iwc); doc = new Document(); @@ -628,7 +199,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalNumBytesChangeOneDoc() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -645,7 +216,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalNumBytesChangeTwoDocs() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -664,7 +235,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalNumBytesChangeTwoSegments() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -684,7 +255,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalNumBytesChangeTwoWriters() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -706,7 +277,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalNumBytesChangeViaAddIndexesDirectory() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -714,7 +285,7 @@ public class TestPointValues extends LuceneTestCase { w.addDocument(doc); w.close(); - Directory dir2 = getDirectory(1); + Directory dir2 = newDirectory(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); w = new IndexWriter(dir2, iwc); doc = new Document(); @@ -729,7 +300,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalNumBytesChangeViaAddIndexesCodecReader() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -737,7 +308,7 @@ public class TestPointValues extends LuceneTestCase { w.addDocument(doc); w.close(); - Directory dir2 = getDirectory(1); + Directory dir2 = newDirectory(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); w = new IndexWriter(dir2, iwc); doc = new Document(); @@ -753,7 +324,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalNumBytesChangeViaAddIndexesSlowCodecReader() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -761,7 +332,7 @@ public class TestPointValues extends LuceneTestCase { w.addDocument(doc); w.close(); - Directory dir2 = getDirectory(1); + Directory dir2 = newDirectory(); iwc = new IndexWriterConfig(new MockAnalyzer(random())); w = new IndexWriter(dir2, iwc); doc = new Document(); @@ -777,7 +348,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalTooManyBytes() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -796,7 +367,7 @@ public class TestPointValues extends LuceneTestCase { } public void testIllegalTooManyDimensions() throws Exception { - Directory dir = getDirectory(1); + Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); IndexWriter w = new IndexWriter(dir, iwc); Document doc = new Document(); @@ -817,449 +388,4 @@ public class TestPointValues extends LuceneTestCase { w.close(); dir.close(); } - - private void doTestRandomBinary(int count) throws Exception { - int numDocs = TestUtil.nextInt(random(), count, count*2); - int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES); - int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS); - - byte[][][] docValues = new byte[numDocs][][]; - - for(int docID=0;docID 0) { - System.arraycopy(docValues[ord][dim], 0, expectedMaxValues[dim], 0, numBytesPerDim); - } - } - } - } - - // 20% of the time we add into a separate directory, then at some point use - // addIndexes to bring the indexed point values to the main directory: - Directory saveDir; - RandomIndexWriter saveW; - int addIndexesAt; - if (random().nextInt(5) == 1) { - saveDir = dir; - saveW = w; - dir = getDirectory(numValues); - if (useRealWriter) { - iwc = new IndexWriterConfig(new MockAnalyzer(random())); - } else { - iwc = newIndexWriterConfig(); - } - iwc.setCodec(getCodec()); - if (expectExceptions) { - MergeScheduler ms = iwc.getMergeScheduler(); - if (ms instanceof ConcurrentMergeScheduler) { - ((ConcurrentMergeScheduler) ms).setSuppressExceptions(); - } - } - w = new RandomIndexWriter(random(), dir, iwc); - addIndexesAt = TestUtil.nextInt(random(), 1, numValues-1); - } else { - saveW = null; - saveDir = null; - addIndexesAt = 0; - } - - try { - - Document doc = null; - int lastID = -1; - for(int ord=0;ord 0) { - //System.out.println(" query_outside_cell"); - return Relation.CELL_OUTSIDE_QUERY; - } else if (NumericUtils.compare(numBytesPerDim, minPacked, dim, queryMin[dim], 0) < 0 || - NumericUtils.compare(numBytesPerDim, maxPacked, dim, queryMax[dim], 0) > 0) { - crosses = true; - } - } - - if (crosses) { - //System.out.println(" query_crosses_cell"); - return Relation.CELL_CROSSES_QUERY; - } else { - //System.out.println(" cell_inside_query"); - return Relation.CELL_INSIDE_QUERY; - } - } - }); - - BitSet expected = new BitSet(); - for(int ord=0;ord 0) { - matches = false; - break; - } - } - - if (matches) { - int id; - if (ids == null) { - id = ord; - } else { - id = ids[ord]; - } - expected.set(id); - } - } - - int limit = Math.max(expected.length(), hits.length()); - int failCount = 0; - int successCount = 0; - for(int id=0;id subs = new ArrayList<>(); - for (LeafReaderContext context : r.leaves()) { - subs.add((CodecReader) context.reader()); - } - if (VERBOSE) { - System.out.println("TEST: now use addIndexes(CodecReader[]) to switch writers"); - } - saveW.addIndexes(subs.toArray(new CodecReader[subs.size()])); - } else { - if (VERBOSE) { - System.out.println("TEST: now use TestUtil.addIndexesSlowly(DirectoryReader[]) to switch writers"); - } - TestUtil.addIndexesSlowly(saveW.w, r); - } - } - } else { - // Add via directory: - if (VERBOSE) { - System.out.println("TEST: now use addIndexes(Directory[]) to switch writers"); - } - w.close(); - saveW.addIndexes(new Directory[] {dir}); - } - w.close(); - dir.close(); - } - - private BigInteger randomBigInt(int numBytes) { - BigInteger x = new BigInteger(numBytes*8-1, random()); - if (random().nextBoolean()) { - x = x.negate(); - } - return x; - } - - private static Directory noVirusChecker(Directory dir) { - if (dir instanceof MockDirectoryWrapper) { - ((MockDirectoryWrapper) dir).setEnableVirusScanner(false); - } - return dir; - } - - private Directory getDirectory(int numPoints) throws IOException { - Directory dir; - if (numPoints > 100000) { - dir = newFSDirectory(createTempDir("TestBKDTree")); - } else { - dir = newDirectory(); - } - noVirusChecker(dir); - //dir = FSDirectory.open(createTempDir()); - return dir; - } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java new file mode 100644 index 00000000000..2dd0b7d1de4 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java @@ -0,0 +1,882 @@ +package org.apache.lucene.index; + +import java.io.IOException; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.List; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.BinaryPoint; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.PointValues.IntersectVisitor; +import org.apache.lucene.index.PointValues.Relation; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.StringHelper; +import org.apache.lucene.util.TestUtil; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Abstract class to do basic tests for a points format. + * NOTE: This test focuses on the points impl, nothing else. + * The [stretch] goal is for this test to be + * so thorough in testing a new PointFormat that if this + * test passes, then all Lucene/Solr tests should also pass. Ie, + * if there is some bug in a given PointFormat that this + * test fails to catch then this test needs to be improved! */ +public abstract class BasePointFormatTestCase extends BaseIndexFileFormatTestCase { + + @Override + protected void addRandomFields(Document doc) { + final int numValues = random().nextInt(3); + for (int i = 0; i < numValues; i++) { + doc.add(new IntPoint("f", random().nextInt())); + } + } + + public void testBasic() throws Exception { + Directory dir = getDirectory(20); + IndexWriterConfig iwc = newIndexWriterConfig(); + iwc.setMergePolicy(newLogMergePolicy()); + IndexWriter w = new IndexWriter(dir, iwc); + byte[] point = new byte[4]; + for(int i=0;i<20;i++) { + Document doc = new Document(); + NumericUtils.intToBytes(i, point, 0); + doc.add(new BinaryPoint("dim", point)); + w.addDocument(doc); + } + w.forceMerge(1); + w.close(); + + DirectoryReader r = DirectoryReader.open(dir); + LeafReader sub = getOnlySegmentReader(r); + PointValues values = sub.getPointValues(); + + // Simple test: make sure intersect can visit every doc: + BitSet seen = new BitSet(); + values.intersect("dim", + new IntersectVisitor() { + @Override + public Relation compare(byte[] minPacked, byte[] maxPacked) { + return Relation.CELL_CROSSES_QUERY; + } + public void visit(int docID) { + throw new IllegalStateException(); + } + public void visit(int docID, byte[] packedValue) { + seen.set(docID); + assertEquals(docID, NumericUtils.bytesToInt(packedValue, 0)); + } + }); + assertEquals(20, seen.cardinality()); + IOUtils.close(r, dir); + } + + public void testMerge() throws Exception { + Directory dir = getDirectory(20); + IndexWriterConfig iwc = newIndexWriterConfig(); + iwc.setMergePolicy(newLogMergePolicy()); + IndexWriter w = new IndexWriter(dir, iwc); + byte[] point = new byte[4]; + for(int i=0;i<20;i++) { + Document doc = new Document(); + NumericUtils.intToBytes(i, point, 0); + doc.add(new BinaryPoint("dim", point)); + w.addDocument(doc); + if (i == 10) { + w.commit(); + } + } + w.forceMerge(1); + w.close(); + + DirectoryReader r = DirectoryReader.open(dir); + LeafReader sub = getOnlySegmentReader(r); + PointValues values = sub.getPointValues(); + + // Simple test: make sure intersect can visit every doc: + BitSet seen = new BitSet(); + values.intersect("dim", + new IntersectVisitor() { + @Override + public Relation compare(byte[] minPacked, byte[] maxPacked) { + return Relation.CELL_CROSSES_QUERY; + } + public void visit(int docID) { + throw new IllegalStateException(); + } + public void visit(int docID, byte[] packedValue) { + seen.set(docID); + assertEquals(docID, NumericUtils.bytesToInt(packedValue, 0)); + } + }); + assertEquals(20, seen.cardinality()); + IOUtils.close(r, dir); + } + + public void testAllPointDocsDeletedInSegment() throws Exception { + Directory dir = getDirectory(20); + IndexWriterConfig iwc = newIndexWriterConfig(); + IndexWriter w = new IndexWriter(dir, iwc); + byte[] point = new byte[4]; + for(int i=0;i<10;i++) { + Document doc = new Document(); + NumericUtils.intToBytes(i, point, 0); + doc.add(new BinaryPoint("dim", point)); + doc.add(new NumericDocValuesField("id", i)); + doc.add(newStringField("x", "x", Field.Store.NO)); + w.addDocument(doc); + } + w.addDocument(new Document()); + w.deleteDocuments(new Term("x", "x")); + if (random().nextBoolean()) { + w.forceMerge(1); + } + w.close(); + DirectoryReader r = DirectoryReader.open(dir); + assertEquals(1, r.numDocs()); + PointValues values = MultiPointValues.get(r); + Bits liveDocs = MultiFields.getLiveDocs(r); + NumericDocValues idValues = MultiDocValues.getNumericValues(r, "id"); + + if (values != null) { + BitSet seen = new BitSet(); + values.intersect("dim", + new IntersectVisitor() { + @Override + public Relation compare(byte[] minPacked, byte[] maxPacked) { + return Relation.CELL_CROSSES_QUERY; + } + public void visit(int docID) { + throw new IllegalStateException(); + } + public void visit(int docID, byte[] packedValue) { + if (liveDocs.get(docID)) { + seen.set(docID); + } + assertEquals(idValues.get(docID), NumericUtils.bytesToInt(packedValue, 0)); + } + }); + assertEquals(0, seen.cardinality()); + } + IOUtils.close(r, dir); + } + + /** Make sure we close open files, delete temp files, etc., on exception */ + public void testWithExceptions() throws Exception { + int numDocs = atLeast(10000); + int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES); + int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS); + + byte[][][] docValues = new byte[numDocs][][]; + + for(int docID=0;docID docValues = new ArrayList<>(); + List docIDs = new ArrayList<>(); + + for(int docID=0;docID 0) { + docValues[docID][theEqualDim] = docValues[0][theEqualDim]; + } + } + + verify(docValues, null, numDims, numBytesPerDim); + } + + // Tests on N-dimensional points where each dimension is a BigInteger + public void testBigIntNDims() throws Exception { + + int numDocs = atLeast(1000); + try (Directory dir = getDirectory(numDocs)) { + int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES); + int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS); + IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())); + // We rely on docIDs not changing: + iwc.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); + BigInteger[][] docs = new BigInteger[numDocs][]; + + for(int docID=0;docID " + values[dim]); + } + } + docs[docID] = values; + Document doc = new Document(); + doc.add(new BinaryPoint("field", bytes)); + w.addDocument(doc); + } + + DirectoryReader r = w.getReader(); + w.close(); + + PointValues dimValues = MultiPointValues.get(r); + + int iters = atLeast(100); + for(int iter=0;iter= 0; + + if (max.compareTo(queryMin[dim]) < 0 || min.compareTo(queryMax[dim]) > 0) { + return Relation.CELL_OUTSIDE_QUERY; + } else if (min.compareTo(queryMin[dim]) < 0 || max.compareTo(queryMax[dim]) > 0) { + crosses = true; + } + } + + if (crosses) { + return Relation.CELL_CROSSES_QUERY; + } else { + return Relation.CELL_INSIDE_QUERY; + } + } + }); + + for(int docID=0;docID 0) { + expected = false; + break; + } + } + boolean actual = hits.get(docID); + assertEquals("docID=" + docID, expected, actual); + } + } + r.close(); + } + } + + public void testRandomBinaryTiny() throws Exception { + doTestRandomBinary(10); + } + + public void testRandomBinaryMedium() throws Exception { + doTestRandomBinary(10000); + } + + @Nightly + public void testRandomBinaryBig() throws Exception { + assumeFalse("too slow with SimpleText", Codec.getDefault().getName().equals("SimpleText")); + doTestRandomBinary(200000); + } + + private void doTestRandomBinary(int count) throws Exception { + int numDocs = TestUtil.nextInt(random(), count, count*2); + int numBytesPerDim = TestUtil.nextInt(random(), 2, PointValues.MAX_NUM_BYTES); + int numDims = TestUtil.nextInt(random(), 1, PointValues.MAX_DIMENSIONS); + + byte[][][] docValues = new byte[numDocs][][]; + + for(int docID=0;docID 0) { + System.arraycopy(docValues[ord][dim], 0, expectedMaxValues[dim], 0, numBytesPerDim); + } + } + } + } + + // 20% of the time we add into a separate directory, then at some point use + // addIndexes to bring the indexed point values to the main directory: + Directory saveDir; + RandomIndexWriter saveW; + int addIndexesAt; + if (random().nextInt(5) == 1) { + saveDir = dir; + saveW = w; + dir = getDirectory(numValues); + if (useRealWriter) { + iwc = new IndexWriterConfig(new MockAnalyzer(random())); + } else { + iwc = newIndexWriterConfig(); + } + if (expectExceptions) { + MergeScheduler ms = iwc.getMergeScheduler(); + if (ms instanceof ConcurrentMergeScheduler) { + ((ConcurrentMergeScheduler) ms).setSuppressExceptions(); + } + } + w = new RandomIndexWriter(random(), dir, iwc); + addIndexesAt = TestUtil.nextInt(random(), 1, numValues-1); + } else { + saveW = null; + saveDir = null; + addIndexesAt = 0; + } + + try { + + Document doc = null; + int lastID = -1; + for(int ord=0;ord 0) { + //System.out.println(" query_outside_cell"); + return Relation.CELL_OUTSIDE_QUERY; + } else if (NumericUtils.compare(numBytesPerDim, minPacked, dim, queryMin[dim], 0) < 0 || + NumericUtils.compare(numBytesPerDim, maxPacked, dim, queryMax[dim], 0) > 0) { + crosses = true; + } + } + + if (crosses) { + //System.out.println(" query_crosses_cell"); + return Relation.CELL_CROSSES_QUERY; + } else { + //System.out.println(" cell_inside_query"); + return Relation.CELL_INSIDE_QUERY; + } + } + }); + + BitSet expected = new BitSet(); + for(int ord=0;ord 0) { + matches = false; + break; + } + } + + if (matches) { + int id; + if (ids == null) { + id = ord; + } else { + id = ids[ord]; + } + expected.set(id); + } + } + + int limit = Math.max(expected.length(), hits.length()); + int failCount = 0; + int successCount = 0; + for(int id=0;id subs = new ArrayList<>(); + for (LeafReaderContext context : r.leaves()) { + subs.add((CodecReader) context.reader()); + } + if (VERBOSE) { + System.out.println("TEST: now use addIndexes(CodecReader[]) to switch writers"); + } + saveW.addIndexes(subs.toArray(new CodecReader[subs.size()])); + } else { + if (VERBOSE) { + System.out.println("TEST: now use TestUtil.addIndexesSlowly(DirectoryReader[]) to switch writers"); + } + TestUtil.addIndexesSlowly(saveW.w, r); + } + } + } else { + // Add via directory: + if (VERBOSE) { + System.out.println("TEST: now use addIndexes(Directory[]) to switch writers"); + } + w.close(); + saveW.addIndexes(new Directory[] {dir}); + } + w.close(); + dir.close(); + } + + private BigInteger randomBigInt(int numBytes) { + BigInteger x = new BigInteger(numBytes*8-1, random()); + if (random().nextBoolean()) { + x = x.negate(); + } + return x; + } + + private static Directory noVirusChecker(Directory dir) { + if (dir instanceof MockDirectoryWrapper) { + ((MockDirectoryWrapper) dir).setEnableVirusScanner(false); + } + return dir; + } + + private Directory getDirectory(int numPoints) throws IOException { + Directory dir; + if (numPoints > 100000) { + dir = newFSDirectory(createTempDir("TestBKDTree")); + } else { + dir = newDirectory(); + } + noVirusChecker(dir); + //dir = FSDirectory.open(createTempDir()); + return dir; + } +} diff --git a/lucene/test-framework/src/test/org/apache/lucene/codecs/asserting/TestAssertingPointFormat.java b/lucene/test-framework/src/test/org/apache/lucene/codecs/asserting/TestAssertingPointFormat.java new file mode 100644 index 00000000000..514f673e510 --- /dev/null +++ b/lucene/test-framework/src/test/org/apache/lucene/codecs/asserting/TestAssertingPointFormat.java @@ -0,0 +1,31 @@ +package org.apache.lucene.codecs.asserting; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.index.BasePointFormatTestCase; + +/** Test AssertingPointFormat directly */ +public class TestAssertingPointFormat extends BasePointFormatTestCase { + private final Codec codec = new AssertingCodec(); + + @Override + protected Codec getCodec() { + return codec; + } +} From c260c9d25190e2a446508631e932a83cbec628f3 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 29 Jan 2016 05:02:32 -0500 Subject: [PATCH 12/21] fix test bug --- .../apache/lucene/codecs/lucene60/TestLucene60PointFormat.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointFormat.java index c154f53e6c9..322640bfda6 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene60/TestLucene60PointFormat.java @@ -41,7 +41,7 @@ public class TestLucene60PointFormat extends BasePointFormatTestCase { if (random().nextBoolean()) { // randomize parameters int maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 500); - double maxMBSortInHeap = 0.1 + (3*random().nextDouble()); + double maxMBSortInHeap = 3.0 + (3*random().nextDouble()); if (VERBOSE) { System.out.println("TEST: using Lucene60PointFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap); } From ff49bd5922fcacdbf35cba4056f164b24640f8ba Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 29 Jan 2016 05:45:54 -0500 Subject: [PATCH 13/21] make SimpleTextPointReader heroic about exceptions too --- .../simpletext/SimpleTextPointReader.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointReader.java index 222805e8aba..ce7fee77a8f 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPointReader.java @@ -33,6 +33,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.bkd.BKDReader; @@ -59,8 +60,10 @@ class SimpleTextPointReader extends PointReader { public SimpleTextPointReader(SegmentReadState readState) throws IOException { // Initialize readers now: - String fileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, readState.segmentSuffix, SimpleTextPointFormat.POINT_EXTENSION); - dataIn = readState.directory.openInput(fileName, IOContext.DEFAULT); + + // Read index: + Map fieldToFileOffset = new HashMap<>(); + String indexFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, readState.segmentSuffix, SimpleTextPointFormat.POINT_INDEX_EXTENSION); try (ChecksumIndexInput in = readState.directory.openChecksumInput(indexFileName, IOContext.DEFAULT)) { readLine(in); @@ -70,10 +73,25 @@ class SimpleTextPointReader extends PointReader { String fieldName = stripPrefix(FIELD_FP_NAME); readLine(in); long fp = parseLong(FIELD_FP); - readers.put(fieldName, initReader(fp)); + fieldToFileOffset.put(fieldName, fp); } SimpleTextUtil.checkFooter(in); } + + boolean success = false; + String fileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, readState.segmentSuffix, SimpleTextPointFormat.POINT_EXTENSION); + dataIn = readState.directory.openInput(fileName, IOContext.DEFAULT); + try { + for(Map.Entry ent : fieldToFileOffset.entrySet()) { + readers.put(ent.getKey(), initReader(ent.getValue())); + } + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + this.readState = readState; } From ddbf3a21687249f3e8481f80839052ad0b8f746b Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 29 Jan 2016 06:16:30 -0500 Subject: [PATCH 14/21] fix wrong comment; add test proving it's wrong; add mixed codec tests --- .../codecs/lucene60/Lucene60PointWriter.java | 16 ++-- .../org/apache/lucene/index/PointValues.java | 8 +- .../apache/lucene/index/TestPointValues.java | 49 +++++++++++- .../lucene/index/BasePointFormatTestCase.java | 74 ++++++++++++++----- 4 files changed, 120 insertions(+), 27 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java index 01491fdcdc8..74acb4d30ee 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene60/Lucene60PointWriter.java @@ -141,11 +141,17 @@ public class Lucene60PointWriter extends PointWriter implements Closeable { for(int i=0;i Date: Fri, 29 Jan 2016 09:12:35 -0500 Subject: [PATCH 15/21] don't test merge stability of point values: our BKD tree impl is not stable on 1D merge --- .../org/apache/lucene/index/BasePointFormatTestCase.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java index 3b19530f432..20bdfb5e208 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java @@ -919,4 +919,10 @@ public abstract class BasePointFormatTestCase extends BaseIndexFileFormatTestCas //dir = FSDirectory.open(createTempDir()); return dir; } + + @Override + public void testMergeStability() { + // suppress this test from base class: merges for BKD trees are not stable because the tree created by merge will have a different + // structure than the tree created by adding points separately + } } From 3141c697e5f85aeabf63513c7c510cd3862bbc81 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 29 Jan 2016 09:18:31 -0500 Subject: [PATCH 16/21] revert --- .../org/apache/lucene/index/BasePointFormatTestCase.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java index 20bdfb5e208..3b19530f432 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java @@ -919,10 +919,4 @@ public abstract class BasePointFormatTestCase extends BaseIndexFileFormatTestCas //dir = FSDirectory.open(createTempDir()); return dir; } - - @Override - public void testMergeStability() { - // suppress this test from base class: merges for BKD trees are not stable because the tree created by merge will have a different - // structure than the tree created by adding points separately - } } From c403083872408b6f70f10b999fc2ae706804a672 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 29 Jan 2016 09:25:48 -0500 Subject: [PATCH 17/21] try again --- .../apache/lucene/index/BaseIndexFileFormatTestCase.java | 5 +++++ .../org/apache/lucene/index/BasePointFormatTestCase.java | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java index b4b6f7d26f9..c53293ccd65 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java @@ -195,6 +195,7 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase { /** The purpose of this test is to make sure that bulk merge doesn't accumulate useless data over runs. */ public void testMergeStability() throws Exception { + assumeTrue("merge is not stable", mergeIsStable()); Directory dir = newDirectory(); if (dir instanceof MockDirectoryWrapper) { // Else, the virus checker may prevent deletion of files and cause @@ -240,6 +241,10 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase { dir2.close(); } + protected boolean mergeIsStable() { + return true; + } + /** Test the accuracy of the ramBytesUsed estimations. */ @Slow public void testRamBytesUsed() throws IOException { diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java index 3b19530f432..2b88d7460f6 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePointFormatTestCase.java @@ -919,4 +919,11 @@ public abstract class BasePointFormatTestCase extends BaseIndexFileFormatTestCas //dir = FSDirectory.open(createTempDir()); return dir; } + + @Override + protected boolean mergeIsStable() { + // suppress this test from base class: merges for BKD trees are not stable because the tree created by merge will have a different + // structure than the tree created by adding points separately + return false; + } } From e6db8ba2149e9733b7ca4d19a90ff9a36c75df1e Mon Sep 17 00:00:00 2001 From: yonik Date: Fri, 29 Jan 2016 10:59:49 -0500 Subject: [PATCH 18/21] SOLR-8532: GraphQuery don't collect edges at maxDepth level --- solr/CHANGES.txt | 3 + .../apache/solr/search/join/GraphQuery.java | 91 +++++++++++-------- .../solr/search/join/GraphTermsCollector.java | 2 +- .../solr/search/join/GraphQueryTest.java | 23 +++++ 4 files changed, 79 insertions(+), 40 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 5ff042f1542..f5c88a3de6f 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -172,6 +172,9 @@ Optimizations count. Also includes change to move to the next non-zero term value when selecting a segment position. (Keith Laban, Steve Bower, Dennis Gove) +* SOLR-8532: Optimize GraphQuery when maxDepth is set by not collecting edges at the maxDepth level. + (Kevin Watters via yonik) + Other Changes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java index 5f9bfd26f24..a31568a096d 100644 --- a/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java +++ b/solr/core/src/java/org/apache/solr/search/join/GraphQuery.java @@ -135,8 +135,8 @@ public class GraphQuery extends Query { SolrIndexSearcher fromSearcher; private float queryNorm = 1.0F; private float queryWeight = 1.0F; - int frontierSize = 0; - public int currentDepth = 0; + private int frontierSize = 0; + private int currentDepth = -1; private Filter filter; private DocSet resultSet; @@ -177,69 +177,82 @@ public class GraphQuery extends Query { * @throws IOException - if a sub search fails... maybe other cases too! :) */ private DocSet getDocSet() throws IOException { - DocSet fromSet = null; - FixedBitSet seedResultBits = null; // Size that the bit set needs to be. int capacity = fromSearcher.getRawReader().maxDoc(); // The bit set to contain the results that match the query. FixedBitSet resultBits = new FixedBitSet(capacity); - // The measure of how deep in the graph we have gone. - currentDepth = 0; + // this holds the result at each level + BitDocSet fromSet = null; + // the root docs if we return root is false + FixedBitSet rootBits = null; // the initial query for the frontier for the first query Query frontierQuery = q; // Find all documents in this graph that are leaf nodes to speed traversal - // TODO: speed this up in the future with HAS_FIELD type queries - BooleanQuery.Builder leafNodeQuery = new BooleanQuery.Builder(); - WildcardQuery edgeQuery = new WildcardQuery(new Term(toField, "*")); - leafNodeQuery.add(edgeQuery, Occur.MUST_NOT); - DocSet leafNodes = fromSearcher.getDocSet(leafNodeQuery.build()); + DocSet leafNodes = resolveLeafNodes(toField); // Start the breadth first graph traversal. + do { - // Create the graph result collector for this level - GraphTermsCollector graphResultCollector = new GraphTermsCollector(toField,capacity, resultBits, leafNodes); - // traverse the level! - fromSearcher.search(frontierQuery, graphResultCollector); - // All edge ids on the frontier. - BytesRefHash collectorTerms = graphResultCollector.getCollectorTerms(); - frontierSize = collectorTerms.size(); - // The resulting doc set from the frontier. - fromSet = graphResultCollector.getDocSet(); - if (seedResultBits == null) { - // grab a copy of the seed bits (these are the "rootNodes") - seedResultBits = ((BitDocSet)fromSet).getBits().clone(); - } - Integer fs = new Integer(frontierSize); - FrontierQuery fq = buildFrontierQuery(collectorTerms, fs); - if (fq == null) { - // in case we get null back, make sure we know we're done at this level. - fq = new FrontierQuery(null, 0); - } - frontierQuery = fq.getQuery(); - frontierSize = fq.getFrontierSize(); - // Add the bits from this level to the result set. - resultBits.or(((BitDocSet)fromSet).getBits()); // Increment how far we have gone in the frontier. currentDepth++; - // Break out if we have reached our max depth - if (currentDepth >= maxDepth && maxDepth != -1) { + // if we are at the max level we don't need the graph terms collector. + // TODO validate that the join case works properly. + if (maxDepth != -1 && currentDepth >= maxDepth) { + // if we've reached the max depth, don't worry about collecting edges. + fromSet = fromSearcher.getDocSetBits(frontierQuery); + // explicitly the frontier size is zero now so we can break + frontierSize = 0; + } else { + // when we're not at the max depth level, we need to collect edges + // Create the graph result collector for this level + GraphTermsCollector graphResultCollector = new GraphTermsCollector(toField,capacity, resultBits, leafNodes); + fromSearcher.search(frontierQuery, graphResultCollector); + fromSet = graphResultCollector.getDocSet(); + // All edge ids on the frontier. + BytesRefHash collectorTerms = graphResultCollector.getCollectorTerms(); + frontierSize = collectorTerms.size(); + // The resulting doc set from the frontier. + FrontierQuery fq = buildFrontierQuery(collectorTerms, frontierSize); + if (fq == null) { + // in case we get null back, make sure we know we're done at this level. + frontierSize = 0; + } else { + frontierQuery = fq.getQuery(); + frontierSize = fq.getFrontierSize(); + } + } + if (currentDepth == 0 && !returnRoot) { + // grab a copy of the root bits but only if we need it. + rootBits = fromSet.getBits(); + } + // Add the bits from this level to the result set. + resultBits.or(fromSet.getBits()); + // test if we discovered any new edges, if not , we're done. + if ((maxDepth != -1 && currentDepth >= maxDepth)) { break; } - // test if we discovered any new edges, if not , we're done. } while (frontierSize > 0); // helper bit set operations on the final result set if (!returnRoot) { - resultBits.andNot(seedResultBits); + resultBits.andNot(rootBits); } + // this is the final resulting filter. BitDocSet resultSet = new BitDocSet(resultBits); // If we only want to return leaf nodes do that here. if (onlyLeafNodes) { return resultSet.intersection(leafNodes); } else { - // create a doc set off the bits that we found. return resultSet; } } + private DocSet resolveLeafNodes(String field) throws IOException { + BooleanQuery.Builder leafNodeQuery = new BooleanQuery.Builder(); + WildcardQuery edgeQuery = new WildcardQuery(new Term(field, "*")); + leafNodeQuery.add(edgeQuery, Occur.MUST_NOT); + DocSet leafNodes = fromSearcher.getDocSet(leafNodeQuery.build()); + return leafNodes; + } + /** Build an automaton to represent the frontier query */ private Automaton buildAutomaton(BytesRefHash termBytesHash) { // need top pass a sorted set of terms to the autn builder (maybe a better way to avoid this?) diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphTermsCollector.java b/solr/core/src/java/org/apache/solr/search/join/GraphTermsCollector.java index 6af3694a248..389721e066b 100644 --- a/solr/core/src/java/org/apache/solr/search/join/GraphTermsCollector.java +++ b/solr/core/src/java/org/apache/solr/search/join/GraphTermsCollector.java @@ -108,7 +108,7 @@ class GraphTermsCollector extends SimpleCollector implements Collector { numHits++; } - public DocSet getDocSet() { + public BitDocSet getDocSet() { if (bits == null) { // TODO: this shouldn't happen bits = new FixedBitSet(maxDoc); diff --git a/solr/core/src/test/org/apache/solr/search/join/GraphQueryTest.java b/solr/core/src/test/org/apache/solr/search/join/GraphQueryTest.java index 4385dcc3e84..1f5de65f701 100644 --- a/solr/core/src/test/org/apache/solr/search/join/GraphQueryTest.java +++ b/solr/core/src/test/org/apache/solr/search/join/GraphQueryTest.java @@ -77,6 +77,29 @@ public class GraphQueryTest extends SolrTestCaseJ4 { qr = createRequest(g4Query); assertQ(qr,"//*[@numFound='2']"); + String g5Query = "{!graph from=\"node_id\" to=\"edge_id\" returnRoot=\"true\" returnOnlyLeaf=\"false\" maxDepth=0}id:doc_8"; + qr = createRequest(g5Query); + assertQ(qr,"//*[@numFound='1']"); + + String g6Query = "{!graph from=\"node_id\" to=\"edge_id\" returnRoot=\"true\" returnOnlyLeaf=\"false\" maxDepth=1}id:doc_8"; + qr = createRequest(g6Query); + assertQ(qr,"//*[@numFound='3']"); + + String g7Query = "{!graph from=\"node_id\" to=\"edge_id\" returnRoot=\"false\" returnOnlyLeaf=\"false\" maxDepth=1}id:doc_8"; + qr = createRequest(g7Query); + assertQ(qr,"//*[@numFound='2']"); + + String g8Query = "{!graph from=\"node_id\" to=\"edge_id\" returnRoot=\"false\" returnOnlyLeaf=\"true\" maxDepth=2}id:doc_8"; + qr = createRequest(g8Query); + assertQ(qr,"//*[@numFound='1']"); + + String g9Query = "{!graph from=\"node_id\" to=\"edge_id\" maxDepth=1}id:doc_1"; + qr = createRequest(g9Query); + assertQ(qr,"//*[@numFound='2']"); + + String g10Query = "{!graph from=\"node_id\" to=\"edge_id\" returnRoot=false maxDepth=1}id:doc_1"; + qr = createRequest(g10Query); + assertQ(qr,"//*[@numFound='1']"); } private SolrQueryRequest createRequest(String query) { From 07c9b2bdad1c072678763993c0ba2ed3bba810bc Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Fri, 29 Jan 2016 11:43:39 -0500 Subject: [PATCH 19/21] Use 2d points too in some of these tests (1d has optimized merge, for example) --- .../src/test/org/apache/lucene/index/TestAddIndexes.java | 6 ++++++ .../src/test/org/apache/lucene/index/TestAtomicUpdate.java | 1 + .../org/apache/lucene/index/TestCodecHoldsOpenFiles.java | 1 + .../org/apache/lucene/index/TestIndexWriterExceptions2.java | 1 + .../org/apache/lucene/index/TestIndexWriterOnDiskFull.java | 2 ++ .../org/apache/lucene/index/TestIndexWriterOnVMError.java | 1 + 6 files changed, 12 insertions(+) diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java index ca6808f80a5..778a7eb7353 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java @@ -172,6 +172,7 @@ public class TestAddIndexes extends LuceneTestCase { doc.add(newStringField("id", "" + (i % 10), Field.Store.NO)); doc.add(newTextField("content", "bbb " + i, Field.Store.NO)); doc.add(new IntPoint("doc", i)); + doc.add(new IntPoint("doc2d", i, i)); doc.add(new NumericDocValuesField("dv", i)); writer.updateDocument(new Term("id", "" + (i%10)), doc); } @@ -207,6 +208,7 @@ public class TestAddIndexes extends LuceneTestCase { doc.add(newStringField("id", "" + (i % 10), Field.Store.NO)); doc.add(newTextField("content", "bbb " + i, Field.Store.NO)); doc.add(new IntPoint("doc", i)); + doc.add(new IntPoint("doc2d", i, i)); doc.add(new NumericDocValuesField("dv", i)); writer.updateDocument(new Term("id", "" + (i%10)), doc); } @@ -245,6 +247,7 @@ public class TestAddIndexes extends LuceneTestCase { doc.add(newStringField("id", "" + (i % 10), Field.Store.NO)); doc.add(newTextField("content", "bbb " + i, Field.Store.NO)); doc.add(new IntPoint("doc", i)); + doc.add(new IntPoint("doc2d", i, i)); doc.add(new NumericDocValuesField("dv", i)); writer.updateDocument(new Term("id", "" + (i%10)), doc); } @@ -519,6 +522,7 @@ public class TestAddIndexes extends LuceneTestCase { Document doc = new Document(); doc.add(newTextField("content", "aaa", Field.Store.NO)); doc.add(new IntPoint("doc", i)); + doc.add(new IntPoint("doc2d", i, i)); doc.add(new NumericDocValuesField("dv", i)); writer.addDocument(doc); } @@ -529,6 +533,7 @@ public class TestAddIndexes extends LuceneTestCase { Document doc = new Document(); doc.add(newTextField("content", "bbb", Field.Store.NO)); doc.add(new IntPoint("doc", i)); + doc.add(new IntPoint("doc2d", i, i)); doc.add(new NumericDocValuesField("dv", i)); writer.addDocument(doc); } @@ -1014,6 +1019,7 @@ public class TestAddIndexes extends LuceneTestCase { doc.add(newTextField("content", "aaa", Field.Store.NO)); doc.add(newTextField("id", "" + (docStart + i), Field.Store.YES)); doc.add(new IntPoint("doc", i)); + doc.add(new IntPoint("doc2d", i, i)); doc.add(new NumericDocValuesField("dv", i)); writer.addDocument(doc); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java b/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java index 1eba37a8278..e801bccbc8a 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java @@ -81,6 +81,7 @@ public class TestAtomicUpdate extends LuceneTestCase { d.add(new StringField("id", Integer.toString(i), Field.Store.YES)); d.add(new TextField("contents", English.intToEnglish(i+10*count), Field.Store.NO)); d.add(new IntPoint("doc", i)); + d.add(new IntPoint("doc2d", i, i)); writer.updateDocument(new Term("id", Integer.toString(i)), d); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java index 0686048b71b..8077545256b 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecHoldsOpenFiles.java @@ -38,6 +38,7 @@ public class TestCodecHoldsOpenFiles extends LuceneTestCase { Document doc = new Document(); doc.add(newField("foo", "bar", TextField.TYPE_NOT_STORED)); doc.add(new IntPoint("doc", i)); + doc.add(new IntPoint("doc2d", i, i)); doc.add(new NumericDocValuesField("dv", i)); w.addDocument(doc); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java index 649170a755d..4c4c496fdec 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions2.java @@ -127,6 +127,7 @@ public class TestIndexWriterExceptions2 extends LuceneTestCase { ft.setStoreTermVectors(true); doc.add(newField("text_vectors", TestUtil.randomAnalysisString(random(), 6, true), ft)); doc.add(new IntPoint("point", random().nextInt())); + doc.add(new IntPoint("point2d", random().nextInt(), random().nextInt())); if (random().nextInt(10) > 0) { // single doc diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java index 8163f8e595b..f75230c8f2d 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java @@ -574,6 +574,7 @@ public class TestIndexWriterOnDiskFull extends LuceneTestCase { doc.add(newTextField("content", "aaa", Field.Store.NO)); doc.add(new NumericDocValuesField("numericdv", 1)); doc.add(new IntPoint("point", 1)); + doc.add(new IntPoint("point2d", 1, 1)); writer.addDocument(doc); } @@ -583,6 +584,7 @@ public class TestIndexWriterOnDiskFull extends LuceneTestCase { doc.add(newTextField("id", "" + index, Field.Store.NO)); doc.add(new NumericDocValuesField("numericdv", 1)); doc.add(new IntPoint("point", 1)); + doc.add(new IntPoint("point2d", 1, 1)); writer.addDocument(doc); } } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnVMError.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnVMError.java index 6c62df2bcfd..91da346f74f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnVMError.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnVMError.java @@ -126,6 +126,7 @@ public class TestIndexWriterOnVMError extends LuceneTestCase { ft.setStoreTermVectors(true); doc.add(newField("text_vectors", TestUtil.randomAnalysisString(random(), 6, true), ft)); doc.add(new IntPoint("point", random().nextInt())); + doc.add(new IntPoint("point2d", random().nextInt(), random().nextInt())); if (random().nextInt(10) > 0) { // single doc From 5d5b082e6d68413c274c8259c235c2d56b14a760 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 29 Jan 2016 15:39:06 -0500 Subject: [PATCH 20/21] catch invalide usage of XXXPoint APIs, when dimensions are mismatched --- .../apache/lucene/document/DoublePoint.java | 6 ++ .../apache/lucene/document/FloatPoint.java | 6 ++ .../org/apache/lucene/document/IntPoint.java | 6 ++ .../org/apache/lucene/document/LongPoint.java | 6 ++ .../apache/lucene/index/TestPointValues.java | 71 +++++++++++++++++++ 5 files changed, 95 insertions(+) diff --git a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java index a7a63e0c34f..a12a0c9853a 100644 --- a/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/DoublePoint.java @@ -42,6 +42,9 @@ public final class DoublePoint extends Field { /** Change the values of this field */ public void setDoubleValues(double... point) { + if (type.pointDimensionCount() != point.length) { + throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions"); + } fieldsData = pack(point); } @@ -52,6 +55,9 @@ public final class DoublePoint extends Field { @Override public Number numericValue() { + if (type.pointDimensionCount() != 1) { + throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value"); + } BytesRef bytes = (BytesRef) fieldsData; assert bytes.length == RamUsageEstimator.NUM_BYTES_LONG; return NumericUtils.sortableLongToDouble(NumericUtils.bytesToLongDirect(bytes.bytes, bytes.offset)); diff --git a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java index a023a4a82b5..00766ef9cb8 100644 --- a/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/FloatPoint.java @@ -42,6 +42,9 @@ public final class FloatPoint extends Field { /** Change the values of this field */ public void setFloatValues(float... point) { + if (type.pointDimensionCount() != point.length) { + throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions"); + } fieldsData = pack(point); } @@ -52,6 +55,9 @@ public final class FloatPoint extends Field { @Override public Number numericValue() { + if (type.pointDimensionCount() != 1) { + throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value"); + } BytesRef bytes = (BytesRef) fieldsData; assert bytes.length == RamUsageEstimator.NUM_BYTES_INT; return NumericUtils.sortableIntToFloat(NumericUtils.bytesToIntDirect(bytes.bytes, bytes.offset)); diff --git a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java index 28f6a555472..ef487066967 100644 --- a/lucene/core/src/java/org/apache/lucene/document/IntPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/IntPoint.java @@ -42,6 +42,9 @@ public final class IntPoint extends Field { /** Change the values of this field */ public void setIntValues(int... point) { + if (type.pointDimensionCount() != point.length) { + throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions"); + } fieldsData = pack(point); } @@ -52,6 +55,9 @@ public final class IntPoint extends Field { @Override public Number numericValue() { + if (type.pointDimensionCount() != 1) { + throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value"); + } BytesRef bytes = (BytesRef) fieldsData; assert bytes.length == RamUsageEstimator.NUM_BYTES_INT; return NumericUtils.bytesToInt(bytes.bytes, bytes.offset); diff --git a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java index 23fddb28f0c..15810d1aafb 100644 --- a/lucene/core/src/java/org/apache/lucene/document/LongPoint.java +++ b/lucene/core/src/java/org/apache/lucene/document/LongPoint.java @@ -42,6 +42,9 @@ public final class LongPoint extends Field { /** Change the values of this field */ public void setLongValues(long... point) { + if (type.pointDimensionCount() != point.length) { + throw new IllegalArgumentException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot change to (incoming) " + point.length + " dimensions"); + } fieldsData = pack(point); } @@ -52,6 +55,9 @@ public final class LongPoint extends Field { @Override public Number numericValue() { + if (type.pointDimensionCount() != 1) { + throw new IllegalStateException("this field (name=" + name + ") uses " + type.pointDimensionCount() + " dimensions; cannot convert to a single numeric value"); + } BytesRef bytes = (BytesRef) fieldsData; assert bytes.length == RamUsageEstimator.NUM_BYTES_LONG; return NumericUtils.bytesToLong(bytes.bytes, bytes.offset); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java index 738109e36ce..ff5fdf883e0 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPointValues.java @@ -21,8 +21,11 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.BinaryPoint; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; @@ -435,4 +438,72 @@ public class TestPointValues extends LuceneTestCase { w.close(); dir.close(); } + + public void testInvalidIntPointUsage() throws Exception { + IntPoint field = new IntPoint("field", 17, 42); + try { + field.setIntValue(14); + fail("did not hit exception"); + } catch (IllegalArgumentException iae) { + // good + } + + try { + field.numericValue(); + fail("did not hit exception"); + } catch (IllegalStateException ise) { + // good + } + } + + public void testInvalidLongPointUsage() throws Exception { + LongPoint field = new LongPoint("field", 17, 42); + try { + field.setLongValue(14); + fail("did not hit exception"); + } catch (IllegalArgumentException iae) { + // good + } + + try { + field.numericValue(); + fail("did not hit exception"); + } catch (IllegalStateException ise) { + // good + } + } + + public void testInvalidFloatPointUsage() throws Exception { + FloatPoint field = new FloatPoint("field", 17, 42); + try { + field.setFloatValue(14); + fail("did not hit exception"); + } catch (IllegalArgumentException iae) { + // good + } + + try { + field.numericValue(); + fail("did not hit exception"); + } catch (IllegalStateException ise) { + // good + } + } + + public void testInvalidDoublePointUsage() throws Exception { + DoublePoint field = new DoublePoint("field", 17, 42); + try { + field.setDoubleValue(14); + fail("did not hit exception"); + } catch (IllegalArgumentException iae) { + // good + } + + try { + field.numericValue(); + fail("did not hit exception"); + } catch (IllegalStateException ise) { + // good + } + } } From 9332b1602cc0f7312fc22a3d088c549299015691 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Fri, 29 Jan 2016 19:09:04 -0500 Subject: [PATCH 21/21] also assert PointValues are the same from LTC.assertReaderEquals --- .../apache/lucene/index/MultiPointValues.java | 4 +- .../apache/lucene/util/LuceneTestCase.java | 64 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiPointValues.java b/lucene/core/src/java/org/apache/lucene/index/MultiPointValues.java index 12282e7c5ab..5dd4fcc111c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiPointValues.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiPointValues.java @@ -23,7 +23,8 @@ import java.util.List; import org.apache.lucene.util.StringHelper; -class MultiPointValues extends PointValues { +/** Merges multiple {@link PointValues} into a single one. */ +public class MultiPointValues extends PointValues { private final List subs; private final List docBases; @@ -33,6 +34,7 @@ class MultiPointValues extends PointValues { this.docBases = docBases; } + /** Returns a {@link PointValues} merging all point values from the provided reader. */ public static PointValues get(IndexReader r) { final List leaves = r.leaves(); final int size = leaves.size(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java index aaab0308903..717583717c8 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java @@ -1888,6 +1888,7 @@ public abstract class LuceneTestCase extends Assert { assertDocValuesEquals(info, leftReader, rightReader); assertDeletedDocsEquals(info, leftReader, rightReader); assertFieldInfosEquals(info, leftReader, rightReader); + assertPointsEquals(info, leftReader, rightReader); } /** @@ -2533,6 +2534,69 @@ public abstract class LuceneTestCase extends Assert { assertEquals(info, left, right); } + // naive silly memory heavy uninversion!! maps docID -> packed values (a Set because a given doc can be multi-valued) + private Map> uninvert(String fieldName, PointValues points) throws IOException { + final Map> docValues = new HashMap<>(); + points.intersect(fieldName, new PointValues.IntersectVisitor() { + @Override + public void visit(int docID) { + throw new UnsupportedOperationException(); + } + + @Override + public void visit(int docID, byte[] packedValue) throws IOException { + if (docValues.containsKey(docID) == false) { + docValues.put(docID, new HashSet()); + } + docValues.get(docID).add(new BytesRef(packedValue.clone())); + } + + @Override + public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + // We pretend our query shape is so hairy that it crosses every single cell: + return PointValues.Relation.CELL_CROSSES_QUERY; + } + }); + return docValues; + } + + public void assertPointsEquals(String info, IndexReader leftReader, IndexReader rightReader) throws IOException { + assertPointsEquals(info, + MultiFields.getMergedFieldInfos(leftReader), + MultiPointValues.get(leftReader), + MultiFields.getMergedFieldInfos(rightReader), + MultiPointValues.get(rightReader)); + } + + public void assertPointsEquals(String info, FieldInfos fieldInfos1, PointValues points1, FieldInfos fieldInfos2, PointValues points2) throws IOException { + for(FieldInfo fieldInfo1 : fieldInfos1) { + if (fieldInfo1.getPointDimensionCount() != 0) { + FieldInfo fieldInfo2 = fieldInfos2.fieldInfo(fieldInfo1.name); + // same dimension count? + assertEquals(info, fieldInfo2.getPointDimensionCount(), fieldInfo2.getPointDimensionCount()); + // same bytes per dimension? + assertEquals(info, fieldInfo2.getPointNumBytes(), fieldInfo2.getPointNumBytes()); + + assertEquals(info + " field=" + fieldInfo1.name, + uninvert(fieldInfo1.name, points1), + uninvert(fieldInfo1.name, points2)); + } + } + + // make sure FieldInfos2 doesn't have any point fields that FieldInfo1 didn't have + for(FieldInfo fieldInfo2 : fieldInfos2) { + if (fieldInfo2.getPointDimensionCount() != 0) { + FieldInfo fieldInfo1 = fieldInfos1.fieldInfo(fieldInfo2.name); + // same dimension count? + assertEquals(info, fieldInfo2.getPointDimensionCount(), fieldInfo1.getPointDimensionCount()); + // same bytes per dimension? + assertEquals(info, fieldInfo2.getPointNumBytes(), fieldInfo1.getPointNumBytes()); + + // we don't need to uninvert and compare here ... we did that in the first loop above + } + } + } + /** Returns true if the file exists (can be opened), false * if it cannot be opened, and (unlike Java's * File.exists) throws IOException if there's some