From c1d30a25ac505f2e513899cd1182def5c8dfff40 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Wed, 7 Dec 2011 23:50:44 +0000 Subject: [PATCH 1/6] LUCENE-3588: Use MapBasedSet git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1211710 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/java/org/apache/lucene/store/MMapDirectory.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lucene/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/src/java/org/apache/lucene/store/MMapDirectory.java index 25696d59a4a..805019b4609 100644 --- a/lucene/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/src/java/org/apache/lucene/store/MMapDirectory.java @@ -26,7 +26,7 @@ import java.nio.channels.ClosedChannelException; // javadoc @link import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; -import java.util.Map; +import java.util.Set; import java.util.WeakHashMap; import java.security.AccessController; @@ -34,6 +34,7 @@ import java.security.PrivilegedExceptionAction; import java.security.PrivilegedActionException; import java.lang.reflect.Method; +import org.apache.lucene.util.MapBackedSet; import org.apache.lucene.util.Constants; /** File-based {@link Directory} implementation that uses @@ -259,7 +260,7 @@ public class MMapDirectory extends FSDirectory { private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex] private boolean isClone = false; - private final Map clones = new WeakHashMap(); + private final Set clones = new MapBackedSet(new WeakHashMap()); MMapIndexInput(String resourceDescription, RandomAccessFile raf, long offset, long length, int chunkSizePower) throws IOException { super(resourceDescription); @@ -430,7 +431,7 @@ public class MMapDirectory extends FSDirectory { // register the new clone in our clone list to clean it up on closing: synchronized(this.clones) { - this.clones.put(clone, Boolean.TRUE); + this.clones.add(clone); } return clone; @@ -449,7 +450,7 @@ public class MMapDirectory extends FSDirectory { // for extra safety unset also all clones' buffers: synchronized(this.clones) { - for (final MMapIndexInput clone : this.clones.keySet()) { + for (final MMapIndexInput clone : this.clones) { assert clone.isClone; clone.unsetBuffers(); } From 97b40049d03edc018e358bc0b2e6c9073cddc7dd Mon Sep 17 00:00:00 2001 From: Grant Ingersoll Date: Thu, 8 Dec 2011 11:09:47 +0000 Subject: [PATCH 2/6] SOLR-1730: Hook in handling of bad cores, failure of QEC to load, add in test git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1211827 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/solr/core/CoreContainer.java | 4 +- .../java/org/apache/solr/core/SolrCore.java | 9 ++-- .../component/QueryElevationComponent.java | 8 ++-- .../solr/servlet/SolrDispatchFilter.java | 2 +- .../solr/conf/solrconfig-elevate.xml | 30 ++++++++++++- .../handler/component/BadComponentTest.java | 43 +++++++++++++++++++ .../QueryElevationComponentTest.java | 9 +++- 7 files changed, 94 insertions(+), 11 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/handler/component/BadComponentTest.java diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index c383b0b8801..a1ee272a7fc 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -215,7 +215,6 @@ public class CoreContainer } /** - * @exception generates an error if you attempt to set this value to false * @deprecated all cores now abort on configuration error regardless of configuration */ @Deprecated @@ -246,6 +245,9 @@ public class CoreContainer } solrConfigFilename = cores.getConfigFile().getName(); + if (cores.cores.isEmpty()){ + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No cores were created, please check the logs for errors"); + } return cores; } diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 3effc622d01..35f17491e1d 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -617,11 +617,14 @@ public final class SolrCore implements SolrInfoMBean { // Finally tell anyone who wants to know resourceLoader.inform( resourceLoader ); resourceLoader.inform( this ); // last call before the latch is released. - } catch (IOException e) { - log.error("", e); + } catch (Throwable e) { + log.error("Error in constructing the core", e); + latch.countDown();//release the latch, otherwise we block trying to do the close. This should be fine, since counting down on a latch of 0 is still fine + //close down the searcher and any other resources, if it exists, as this is not recoverable + close(); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, e, false); } finally { - // allow firstSearcher events to fire + // allow firstSearcher events to fire and make sure it is released latch.countDown(); } diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java index 0fee7ad6176..e0e66d0127e 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java @@ -199,7 +199,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore elevationCache.put(null, loadElevationMap( cfg )); } } - + //in other words, we think this is in the data dir, not the conf dir if (!exists){ // preload the first data RefCounted searchHolder = null; @@ -215,10 +215,10 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore } catch( Exception ex ) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, - "Error initializing QueryElevationComponent.", ex ); + "Error initializing QueryElevationComponent.", ex, false ); } } - + //get the elevation map from the data dir Map getElevationMap( IndexReader reader, SolrCore core ) throws Exception { synchronized( elevationCache ) { @@ -242,7 +242,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore return map; } } - + //load up the elevation map private Map loadElevationMap( Config cfg ) throws IOException { XPath xpath = XPathFactory.newInstance().newXPath(); diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index bf002811937..e4c0ff6c93c 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -96,7 +96,7 @@ public class SolrDispatchFilter implements Filter } catch( Throwable t ) { // catch this so our filter still works - log.error( "Could not start Solr. Check solr/home property", t); + log.error( "Could not start Solr. Check solr/home property and the logs", t); SolrConfig.severeErrors.add( t ); SolrCore.log( t ); } diff --git a/solr/core/src/test-files/solr/conf/solrconfig-elevate.xml b/solr/core/src/test-files/solr/conf/solrconfig-elevate.xml index 63e0b0f2881..669cdd2f949 100644 --- a/solr/core/src/test-files/solr/conf/solrconfig-elevate.xml +++ b/solr/core/src/test-files/solr/conf/solrconfig-elevate.xml @@ -185,7 +185,17 @@ string - elevate.xml + ${elevate.file:elevate.xml} + + + + + + string + ${elevate.data.file:elevate-data.xml} @@ -197,6 +207,24 @@ + + + explicit + + + dataElevate + + + + + diff --git a/solr/core/src/test/org/apache/solr/handler/component/BadComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/BadComponentTest.java new file mode 100644 index 00000000000..9471162ebf1 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/component/BadComponentTest.java @@ -0,0 +1,43 @@ +package org.apache.solr.handler.component; + + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.junit.Test; + +/** + * SOLR-1730, tests what happens when a component fails to initialize properly + * + **/ +public class BadComponentTest extends SolrTestCaseJ4{ + @Test + public void testBadElevate() throws Exception { + try { + System.setProperty("elevate.file", "foo.xml"); + initCore("solrconfig-elevate.xml", "schema12.xml"); + assertTrue(false); + } catch (Throwable e) { + log.error("Exception", e); + assertTrue(true); + } finally { + System.clearProperty("elevate.file"); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java index d1170c6e8e4..ffc8b12a2d3 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java @@ -30,6 +30,7 @@ import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.QueryElevationParams; +import org.apache.solr.common.util.FileUtils; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.handler.component.QueryElevationComponent.ElevationObj; @@ -45,7 +46,13 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - initCore("solrconfig-elevate.xml","schema12.xml"); + //write out elevate-data.xml to the Data dir first by copying it from conf, which we know exists, this way we can test both conf and data configurations + createTempDir(); + File parent = new File(TEST_HOME(), "conf"); + File elevateFile = new File(parent, "elevate.xml"); + File elevateDataFile = new File(dataDir, "elevate-data.xml"); + FileUtils.copyFile(elevateFile, elevateDataFile); + initCore("solrconfig-elevate.xml", "schema12.xml"); } @Before From ee9c4b2920aaa78bddebb1afe6377474fa343877 Mon Sep 17 00:00:00 2001 From: Grant Ingersoll Date: Thu, 8 Dec 2011 11:25:06 +0000 Subject: [PATCH 3/6] SOLR-1730: add changes git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1211839 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index cc73fe185f1..b2728b66948 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -288,6 +288,9 @@ Bug Fixes * SOLR-2509: StringIndexOutOfBoundsException in the spellchecker collate when the term contains a hyphen. (Thomas Gambier caught the bug, Steffen Godskesen did the patch, via Erick Erickson) + +* SOLR-1730: Made it clearer when a core failed to load as well as better logging when the + QueryElevationComponent fails to properly initialize (gsingers) Other Changes ---------------------- From f7cd0e2557bb05476dd01cae73a4cd03dd5d798a Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 8 Dec 2011 13:54:05 +0000 Subject: [PATCH 4/6] FieldValueFitler should expose the field it uses git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1211887 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/search/FieldValueFilter.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lucene/src/java/org/apache/lucene/search/FieldValueFilter.java b/lucene/src/java/org/apache/lucene/search/FieldValueFilter.java index eb592aec3f7..14289c69c79 100644 --- a/lucene/src/java/org/apache/lucene/search/FieldValueFilter.java +++ b/lucene/src/java/org/apache/lucene/search/FieldValueFilter.java @@ -56,6 +56,22 @@ public class FieldValueFilter extends Filter { this.field = field; this.negate = negate; } + + /** + * Returns the field this filter is applied on. + * @return the field this filter is applied on. + */ + public String field() { + return field; + } + + /** + * Returns true iff this filter is negated, otherwise false + * @return true iff this filter is negated, otherwise false + */ + public boolean negate() { + return negate; + } @Override public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) From be741b12531388a47ed18d76dd8f0db90a47a30d Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 8 Dec 2011 16:32:54 +0000 Subject: [PATCH 5/6] LUCENE-3621: switch appendingcodec to use appending blocktree git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1211962 13f79535-47bb-0310-9956-ffa450edef68 --- .../index/codecs/BlockTreeTermsReader.java | 12 +-- .../index/codecs/BlockTreeTermsWriter.java | 37 +++++----- .../appending/AppendingPostingsFormat.java | 73 +++++-------------- .../appending/AppendingTermsDictReader.java | 55 -------------- .../appending/AppendingTermsDictWriter.java | 47 ------------ .../appending/AppendingTermsIndexWriter.java | 45 ------------ ...xReader.java => AppendingTermsReader.java} | 34 ++++++--- .../appending/AppendingTermsWriter.java | 64 ++++++++++++++++ 8 files changed, 130 insertions(+), 237 deletions(-) delete mode 100644 lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java delete mode 100644 lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictWriter.java delete mode 100644 lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexWriter.java rename lucene/src/java/org/apache/lucene/index/codecs/appending/{AppendingTermsIndexReader.java => AppendingTermsReader.java} (57%) create mode 100644 lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsWriter.java diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java index 940b87e8511..2da3948181c 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java @@ -166,16 +166,16 @@ public class BlockTreeTermsReader extends FieldsProducer { } protected void readHeader(IndexInput input) throws IOException { - CodecUtil.checkHeader(input, BlockTreeTermsWriter.CODEC_NAME, - BlockTreeTermsWriter.VERSION_START, - BlockTreeTermsWriter.VERSION_CURRENT); + CodecUtil.checkHeader(input, BlockTreeTermsWriter.TERMS_CODEC_NAME, + BlockTreeTermsWriter.TERMS_VERSION_START, + BlockTreeTermsWriter.TERMS_VERSION_CURRENT); dirOffset = input.readLong(); } protected void readIndexHeader(IndexInput input) throws IOException { - CodecUtil.checkHeader(input, BlockTreeTermsWriter.CODEC_NAME, - BlockTreeTermsWriter.VERSION_START, - BlockTreeTermsWriter.VERSION_CURRENT); + CodecUtil.checkHeader(input, BlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME, + BlockTreeTermsWriter.TERMS_INDEX_VERSION_START, + BlockTreeTermsWriter.TERMS_INDEX_VERSION_CURRENT); indexDirOffset = input.readLong(); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java index 27f1d525c3a..86fff64456b 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java @@ -99,18 +99,21 @@ public class BlockTreeTermsWriter extends FieldsConsumer { static final int OUTPUT_FLAG_IS_FLOOR = 0x1; static final int OUTPUT_FLAG_HAS_TERMS = 0x2; - final static String CODEC_NAME = "BLOCK_TREE_TERMS_DICT"; - - // Initial format - public static final int VERSION_START = 0; - - public static final int VERSION_CURRENT = VERSION_START; - /** Extension of terms file */ static final String TERMS_EXTENSION = "tim"; - static final String TERMS_INDEX_EXTENSION = "tip"; + final static String TERMS_CODEC_NAME = "BLOCK_TREE_TERMS_DICT"; + // Initial format + public static final int TERMS_VERSION_START = 0; + public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_START; - protected final IndexOutput out; + /** Extension of terms index file */ + static final String TERMS_INDEX_EXTENSION = "tip"; + final static String TERMS_INDEX_CODEC_NAME = "BLOCK_TREE_TERMS_INDEX"; + // Initial format + public static final int TERMS_INDEX_VERSION_START = 0; + public static final int TERMS_INDEX_VERSION_CURRENT = TERMS_INDEX_VERSION_START; + + private final IndexOutput out; private final IndexOutput indexOut; final int minItemsInBlock; final int maxItemsInBlock; @@ -178,22 +181,22 @@ public class BlockTreeTermsWriter extends FieldsConsumer { } protected void writeHeader(IndexOutput out) throws IOException { - CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); + CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT); out.writeLong(0); // leave space for end index pointer } protected void writeIndexHeader(IndexOutput out) throws IOException { - CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); + CodecUtil.writeHeader(out, TERMS_INDEX_CODEC_NAME, TERMS_INDEX_VERSION_CURRENT); out.writeLong(0); // leave space for end index pointer } - protected void writeTrailer(long dirStart) throws IOException { - out.seek(CodecUtil.headerLength(CODEC_NAME)); + protected void writeTrailer(IndexOutput out, long dirStart) throws IOException { + out.seek(CodecUtil.headerLength(TERMS_CODEC_NAME)); out.writeLong(dirStart); } - protected void writeIndexTrailer(long dirStart) throws IOException { - indexOut.seek(CodecUtil.headerLength(CODEC_NAME)); + protected void writeIndexTrailer(IndexOutput indexOut, long dirStart) throws IOException { + indexOut.seek(CodecUtil.headerLength(TERMS_INDEX_CODEC_NAME)); indexOut.writeLong(dirStart); } @@ -935,8 +938,8 @@ public class BlockTreeTermsWriter extends FieldsConsumer { indexOut.writeVLong(field.indexStartFP); } } - writeTrailer(dirStart); - writeIndexTrailer(indexDirStart); + writeTrailer(out, dirStart); + writeIndexTrailer(indexOut, indexDirStart); } catch (IOException ioe2) { ioe = ioe2; } finally { diff --git a/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java index e2dd3148452..ff96c33957b 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java @@ -23,19 +23,16 @@ import java.util.Set; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.BlockTreeTermsReader; +import org.apache.lucene.index.codecs.BlockTreeTermsWriter; import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.BlockTermsReader; -import org.apache.lucene.index.codecs.TermsIndexReaderBase; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; /** * Appending postings impl @@ -48,72 +45,39 @@ class AppendingPostingsFormat extends PostingsFormat { } @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) - throws IOException { + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { PostingsWriterBase docsWriter = new Lucene40PostingsWriter(state); boolean success = false; - AppendingTermsIndexWriter indexWriter = null; try { - indexWriter = new AppendingTermsIndexWriter(state); + FieldsConsumer ret = new AppendingTermsWriter(state, docsWriter, BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); success = true; + return ret; } finally { if (!success) { docsWriter.close(); } } - success = false; - try { - FieldsConsumer ret = new AppendingTermsDictWriter(indexWriter, state, docsWriter); - success = true; - return ret; - } finally { - if (!success) { - try { - docsWriter.close(); - } finally { - indexWriter.close(); - } - } - } } @Override - public FieldsProducer fieldsProducer(SegmentReadState state) - throws IOException { - PostingsReaderBase docsReader = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); - TermsIndexReaderBase indexReader; - + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase postings = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + boolean success = false; try { - indexReader = new AppendingTermsIndexReader(state.dir, - state.fieldInfos, - state.segmentInfo.name, - state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator(), - state.segmentSuffix, state.context); - success = true; - } finally { - if (!success) { - docsReader.close(); - } - } - success = false; - try { - FieldsProducer ret = new AppendingTermsDictReader(indexReader, - state.dir, state.fieldInfos, state.segmentInfo.name, - docsReader, - state.context, - Lucene40PostingsFormat.TERMS_CACHE_SIZE, - state.segmentSuffix); + FieldsProducer ret = new AppendingTermsReader( + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postings, + state.context, + state.segmentSuffix, + state.termsIndexDivisor); success = true; return ret; } finally { if (!success) { - try { - docsReader.close(); - } finally { - indexReader.close(); - } + postings.close(); } } } @@ -122,7 +86,6 @@ class AppendingPostingsFormat extends PostingsFormat { public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); - BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files); } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java b/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java deleted file mode 100644 index 742de0b1385..00000000000 --- a/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.apache.lucene.index.codecs.appending; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.BlockTermsReader; -import org.apache.lucene.index.codecs.BlockTermsWriter; -import org.apache.lucene.index.codecs.TermsIndexReaderBase; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.CodecUtil; - -public class AppendingTermsDictReader extends BlockTermsReader { - - public AppendingTermsDictReader(TermsIndexReaderBase indexReader, - Directory dir, FieldInfos fieldInfos, String segment, - PostingsReaderBase postingsReader, IOContext context, - int termsCacheSize, String segmentSuffix) throws IOException { - super(indexReader, dir, fieldInfos, segment, postingsReader, context, - termsCacheSize, segmentSuffix); - } - - @Override - protected void readHeader(IndexInput in) throws IOException { - CodecUtil.checkHeader(in, AppendingTermsDictWriter.CODEC_NAME, - BlockTermsWriter.VERSION_START, BlockTermsWriter.VERSION_CURRENT); - } - - @Override - protected void seekDir(IndexInput in, long dirOffset) throws IOException { - in.seek(in.length() - Long.SIZE / 8); - long offset = in.readLong(); - in.seek(offset); - } - -} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictWriter.java deleted file mode 100644 index 1e595cdd5ab..00000000000 --- a/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictWriter.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.apache.lucene.index.codecs.appending; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.BlockTermsWriter; -import org.apache.lucene.index.codecs.TermsIndexWriterBase; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.CodecUtil; - -public class AppendingTermsDictWriter extends BlockTermsWriter { - final static String CODEC_NAME = "APPENDING_TERMS_DICT"; - - public AppendingTermsDictWriter(TermsIndexWriterBase indexWriter, - SegmentWriteState state, PostingsWriterBase postingsWriter) - throws IOException { - super(indexWriter, state, postingsWriter); - } - - @Override - protected void writeHeader(IndexOutput out) throws IOException { - CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); - } - - @Override - protected void writeTrailer(long dirStart) throws IOException { - out.writeLong(dirStart); - } -} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexWriter.java deleted file mode 100644 index d805c224d54..00000000000 --- a/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexWriter.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.apache.lucene.index.codecs.appending; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.CodecUtil; - -public class AppendingTermsIndexWriter extends FixedGapTermsIndexWriter { - final static String CODEC_NAME = "APPENDING_TERMS_INDEX"; - final static int VERSION_START = 0; - final static int VERSION_CURRENT = VERSION_START; - - public AppendingTermsIndexWriter(SegmentWriteState state) throws IOException { - super(state); - } - - @Override - protected void writeHeader(IndexOutput out) throws IOException { - CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); - } - - @Override - protected void writeTrailer(long dirStart) throws IOException { - out.writeLong(dirStart); - } -} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java b/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsReader.java similarity index 57% rename from lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java rename to lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsReader.java index ed393ad85da..2e6292b70c6 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsReader.java @@ -1,6 +1,6 @@ package org.apache.lucene.index.codecs.appending; -/* +/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,30 +18,40 @@ package org.apache.lucene.index.codecs.appending; */ import java.io.IOException; -import java.util.Comparator; import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; +import org.apache.lucene.index.codecs.BlockTreeTermsReader; +import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CodecUtil; -public class AppendingTermsIndexReader extends FixedGapTermsIndexReader { +/** + * Reads append-only terms from {@link AppendingTermsWriter} + * @lucene.experimental + */ +public class AppendingTermsReader extends BlockTreeTermsReader { - public AppendingTermsIndexReader(Directory dir, FieldInfos fieldInfos, - String segment, int indexDivisor, Comparator termComp, String segmentSuffix, IOContext context) - throws IOException { - super(dir, fieldInfos, segment, indexDivisor, termComp, segmentSuffix, context); + public AppendingTermsReader(Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, + IOContext ioContext, String segmentSuffix, int indexDivisor) throws IOException { + super(dir, fieldInfos, segment, postingsReader, ioContext, segmentSuffix, indexDivisor); } - + @Override protected void readHeader(IndexInput input) throws IOException { - CodecUtil.checkHeader(input, AppendingTermsIndexWriter.CODEC_NAME, - AppendingTermsIndexWriter.VERSION_START, AppendingTermsIndexWriter.VERSION_START); + CodecUtil.checkHeader(input, AppendingTermsWriter.TERMS_CODEC_NAME, + AppendingTermsWriter.TERMS_VERSION_START, + AppendingTermsWriter.TERMS_VERSION_CURRENT); } + @Override + protected void readIndexHeader(IndexInput input) throws IOException { + CodecUtil.checkHeader(input, AppendingTermsWriter.TERMS_INDEX_CODEC_NAME, + AppendingTermsWriter.TERMS_INDEX_VERSION_START, + AppendingTermsWriter.TERMS_INDEX_VERSION_CURRENT); + } + @Override protected void seekDir(IndexInput input, long dirOffset) throws IOException { input.seek(input.length() - Long.SIZE / 8); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsWriter.java new file mode 100644 index 00000000000..409e527ca29 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsWriter.java @@ -0,0 +1,64 @@ +package org.apache.lucene.index.codecs.appending; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.BlockTreeTermsWriter; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.CodecUtil; + +/** + * Append-only version of {@link BlockTreeTermsWriter} + * @lucene.experimental + */ +public class AppendingTermsWriter extends BlockTreeTermsWriter { + final static String TERMS_CODEC_NAME = "APPENDING_TERMS_DICT"; + final static int TERMS_VERSION_START = 0; + final static int TERMS_VERSION_CURRENT = TERMS_VERSION_START; + + final static String TERMS_INDEX_CODEC_NAME = "APPENDING_TERMS_INDEX"; + final static int TERMS_INDEX_VERSION_START = 0; + final static int TERMS_INDEX_VERSION_CURRENT = TERMS_INDEX_VERSION_START; + + public AppendingTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter, int minItemsInBlock, int maxItemsInBlock) throws IOException { + super(state, postingsWriter, minItemsInBlock, maxItemsInBlock); + } + + @Override + protected void writeHeader(IndexOutput out) throws IOException { + CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT); + } + + @Override + protected void writeIndexHeader(IndexOutput out) throws IOException { + CodecUtil.writeHeader(out, TERMS_INDEX_CODEC_NAME, TERMS_INDEX_VERSION_CURRENT); + } + + @Override + protected void writeTrailer(IndexOutput out, long dirStart) throws IOException { + out.writeLong(dirStart); + } + + @Override + protected void writeIndexTrailer(IndexOutput indexOut, long dirStart) throws IOException { + indexOut.writeLong(dirStart); + } +} From b3a07fad1fa43836c92f16662fc0d51e69d942df Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 8 Dec 2011 17:18:19 +0000 Subject: [PATCH 6/6] LUCENE-3496: Replaced generating simple string with realistic unicode strings. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1211993 13f79535-47bb-0310-9956-ffa450edef68 --- .../test/org/apache/lucene/search/grouping/TestGrouping.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java b/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java index eb605cae285..216eea907d9 100644 --- a/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java +++ b/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java @@ -672,8 +672,7 @@ public class TestGrouping extends LuceneTestCase { do { // B/c of DV based impl we can't see the difference between an empty string and a null value. // For that reason we don't generate empty string groups. -// randomValue = _TestUtil.randomRealisticUnicodeString(random); - randomValue = _TestUtil.randomSimpleString(random); + randomValue = _TestUtil.randomRealisticUnicodeString(random); } while ("".equals(randomValue)); groups.add(new BytesRef(randomValue));