From 02fe1c42c9f1d7b8724e34204ac3fd2c5d5cb27e Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Mon, 21 Jan 2013 17:39:30 +0000 Subject: [PATCH] clear some nocommits git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1436509 13f79535-47bb-0310-9956-ffa450edef68 --- .../codecs/diskdv/DiskDocValuesConsumer.java | 1 - .../codecs/memory/MemoryDocValuesFormat.java | 160 ------------------ .../codecs/simpletext/SimpleTextCodec.java | 10 +- .../simpletext/SimpleTextDocValuesFormat.java | 4 +- .../simpletext/SimpleTextNormsFormat.java | 20 +-- .../org.apache.lucene.codecs.DocValuesFormat | 1 - .../lucene/codecs/lucene40/Lucene40Codec.java | 4 +- .../perfield/PerFieldDocValuesFormat.java | 8 +- .../lucene/index/SegmentCoreReaders.java | 80 ++++----- .../org/apache/lucene/TestDemoDocValue.java | 3 +- .../org/apache/lucene/index/RandomCodec.java | 6 +- 11 files changed, 53 insertions(+), 244 deletions(-) delete mode 100644 lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java index f04f1088014..9777c605097 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java @@ -31,7 +31,6 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedInts.FormatAndBits; -// nocommit fix exception handling (make sure tests find problems first) class DiskDocValuesConsumer extends DocValuesConsumer { final IndexOutput data, meta; final int maxDoc; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java deleted file mode 100644 index 921eda58a12..00000000000 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java +++ /dev/null @@ -1,160 +0,0 @@ -package org.apache.lucene.codecs.memory; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.codecs.DocValuesConsumer; -import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.codecs.DocValuesFormat; -import org.apache.lucene.codecs.simpletext.SimpleTextDocValuesFormat.SimpleTextDocValuesReader; -import org.apache.lucene.codecs.simpletext.SimpleTextDocValuesFormat.SimpleTextDocValuesWriter; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.packed.PackedInts; - -/** Indexes doc values to disk and loads them in RAM at - * search time. */ - -// nocommit: nuke this wrapper and just make a nice impl for 4.1 (e.g. FST for sortedbytes) -public class MemoryDocValuesFormat extends DocValuesFormat { - - public MemoryDocValuesFormat() { - super("Memory"); - } - - @Override - public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - // nocommit use a more efficient format ;): - return new SimpleTextDocValuesWriter(state, "dat"); - } - - // nocommit the get's of this thing need to use a map. its returning new ram instances - // per-thread! - - @Override - public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException { - final int maxDoc = state.segmentInfo.getDocCount(); - final DocValuesProducer producer = new SimpleTextDocValuesReader(state, "dat"); - - return new DocValuesProducer() { - - @Override - public NumericDocValues getNumeric(FieldInfo field) throws IOException { - NumericDocValues valuesIn = producer.getNumeric(field); - - long minValue = Long.MAX_VALUE; - long maxValue = Long.MIN_VALUE; - for(int docID=0;docID .len; // otherwise this is a normal simple doc values file: super(state, NORMS_SEG_EXTENSION); @@ -75,8 +69,8 @@ public class SimpleTextNormsFormat extends NormsFormat { * * @lucene.experimental */ - public static class SimpleTextSimpleNormsConsumer extends SimpleTextDocValuesWriter { - public SimpleTextSimpleNormsConsumer(SegmentWriteState state) throws IOException { + public static class SimpleTextNormsConsumer extends SimpleTextDocValuesWriter { + public SimpleTextNormsConsumer(SegmentWriteState state) throws IOException { // All we do is change the extension from .dat -> .len; // otherwise this is a normal simple doc values file: super(state, NORMS_SEG_EXTENSION); diff --git a/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat b/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat index 5103c529355..e289c4d0966 100644 --- a/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat +++ b/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat @@ -14,5 +14,4 @@ # limitations under the License. org.apache.lucene.codecs.diskdv.DiskDocValuesFormat -org.apache.lucene.codecs.memory.MemoryDocValuesFormat org.apache.lucene.codecs.simpletext.SimpleTextDocValuesFormat diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java index b893558745e..bb4427f05ec 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java @@ -87,7 +87,7 @@ public final class Lucene40Codec extends Codec { return infosFormat; } - // nocommit need a read-only Lucene40SimpleDVFormat + // nocommit need a read-only Lucene40DVFormat / read-write in the impersonator private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Disk"); @Override @@ -96,7 +96,7 @@ public final class Lucene40Codec extends Codec { return defaultDVFormat; } - // nocommit need a read-only Lucene40SimpleNormsFormat: + // nocommit need a read-only Lucene40NormsFormat / read-write in the impersonator private final NormsFormat simpleNormsFormat = new Lucene41NormsFormat(); @Override diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java index 9f36d5a7178..9e678803047 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java @@ -78,7 +78,7 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat { return new FieldsWriter(state); } - static class SimpleDVConsumerAndSuffix implements Closeable { + static class ConsumerAndSuffix implements Closeable { DocValuesConsumer consumer; int suffix; @@ -90,7 +90,7 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat { private class FieldsWriter extends DocValuesConsumer { - private final Map formats = new HashMap(); + private final Map formats = new HashMap(); private final Map suffixes = new HashMap(); private final SegmentWriteState segmentWriteState; @@ -126,7 +126,7 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat { Integer suffix; - SimpleDVConsumerAndSuffix consumer = formats.get(format); + ConsumerAndSuffix consumer = formats.get(format); if (consumer == null) { // First time we are seeing this format; create a new instance @@ -142,7 +142,7 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat { final String segmentSuffix = getFullSegmentSuffix(field.name, segmentWriteState.segmentSuffix, getSuffix(formatName, Integer.toString(suffix))); - consumer = new SimpleDVConsumerAndSuffix(); + consumer = new ConsumerAndSuffix(); consumer.consumer = format.fieldsConsumer(new SegmentWriteState(segmentWriteState, segmentSuffix)); consumer.suffix = suffix; formats.put(format, consumer); diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java b/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java index 92b5dd70819..439a48b61d4 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java @@ -54,8 +54,8 @@ final class SegmentCoreReaders { final FieldInfos fieldInfos; final FieldsProducer fields; - final DocValuesProducer simpleDVProducer; - final DocValuesProducer simpleNormsProducer; + final DocValuesProducer dvProducer; + final DocValuesProducer normsProducer; final int termsIndexDivisor; @@ -67,7 +67,7 @@ final class SegmentCoreReaders { // TODO: make a single thread local w/ a // Thingy class holding fieldsReader, termVectorsReader, - // simpleNormsProducer, simpleDVProducer + // normsProducer, dvProducer final CloseableThreadLocal fieldsReaderLocal = new CloseableThreadLocal() { @Override @@ -83,14 +83,14 @@ final class SegmentCoreReaders { } }; - final CloseableThreadLocal> simpleDocValuesLocal = new CloseableThreadLocal>() { + final CloseableThreadLocal> docValuesLocal = new CloseableThreadLocal>() { @Override protected Map initialValue() { return new HashMap(); } }; - final CloseableThreadLocal> simpleNormsLocal = new CloseableThreadLocal>() { + final CloseableThreadLocal> normsLocal = new CloseableThreadLocal>() { @Override protected Map initialValue() { return new HashMap(); @@ -129,26 +129,19 @@ final class SegmentCoreReaders { // ask codec for its Norms: // TODO: since we don't write any norms file if there are no norms, // kinda jaky to assume the codec handles the case of no norms file at all gracefully?! - // nocommit shouldn't need null check: - assert codec.docValuesFormat() != null; - if (codec.docValuesFormat() != null) { - if (fieldInfos.hasDocValues()) { - simpleDVProducer = codec.docValuesFormat().fieldsProducer(segmentReadState); - } else { - simpleDVProducer = null; - } + + if (fieldInfos.hasDocValues()) { + dvProducer = codec.docValuesFormat().fieldsProducer(segmentReadState); + assert dvProducer != null; } else { - simpleDVProducer = null; + dvProducer = null; } - // nocommit shouldn't need null check: - if (codec.normsFormat() != null) { - if (fieldInfos.hasNorms()) { - simpleNormsProducer = codec.normsFormat().normsProducer(segmentReadState); - } else { - simpleNormsProducer = null; - } + + if (fieldInfos.hasNorms()) { + normsProducer = codec.normsFormat().normsProducer(segmentReadState); + assert normsProducer != null; } else { - simpleNormsProducer = null; + normsProducer = null; } fieldsReaderOrig = si.info.getCodec().storedFieldsFormat().fieldsReader(cfsDir, si.info, fieldInfos, context); @@ -192,16 +185,13 @@ final class SegmentCoreReaders { return null; } - // nocommit change to assert != null!! - if (simpleDVProducer == null) { - return null; - } + assert dvProducer != null; - Map dvFields = simpleDocValuesLocal.get(); + Map dvFields = docValuesLocal.get(); NumericDocValues dvs = (NumericDocValues) dvFields.get(field); if (dvs == null) { - dvs = simpleDVProducer.getNumeric(fi); + dvs = dvProducer.getNumeric(fi); dvFields.put(field, dvs); } @@ -223,16 +213,13 @@ final class SegmentCoreReaders { return null; } - // nocommit change to assert != null!! - if (simpleDVProducer == null) { - return null; - } + assert dvProducer != null; - Map dvFields = simpleDocValuesLocal.get(); + Map dvFields = docValuesLocal.get(); BinaryDocValues dvs = (BinaryDocValues) dvFields.get(field); if (dvs == null) { - dvs = simpleDVProducer.getBinary(fi); + dvs = dvProducer.getBinary(fi); dvFields.put(field, dvs); } @@ -254,18 +241,13 @@ final class SegmentCoreReaders { return null; } - assert simpleDVProducer != null; + assert dvProducer != null; - // nocommit change to assert != null!! - if (simpleDVProducer == null) { - return null; - } - - Map dvFields = simpleDocValuesLocal.get(); + Map dvFields = docValuesLocal.get(); SortedDocValues dvs = (SortedDocValues) dvFields.get(field); if (dvs == null) { - dvs = simpleDVProducer.getSorted(fi); + dvs = dvProducer.getSorted(fi); dvFields.put(field, dvs); } @@ -281,16 +263,14 @@ final class SegmentCoreReaders { if (!fi.hasNorms()) { return null; } - // nocommit change to assert != null!! - if (simpleNormsProducer == null) { - return null; - } + + assert normsProducer != null; - Map normFields = simpleNormsLocal.get(); + Map normFields = normsLocal.get(); NumericDocValues norms = (NumericDocValues) normFields.get(field); if (norms == null) { - norms = simpleNormsProducer.getNumeric(fi); + norms = normsProducer.getNumeric(fi); normFields.put(field, norms); } @@ -299,8 +279,8 @@ final class SegmentCoreReaders { void decRef() throws IOException { if (ref.decrementAndGet() == 0) { - IOUtils.close(termVectorsLocal, fieldsReaderLocal, simpleDocValuesLocal, simpleNormsLocal, fields, simpleDVProducer, - termVectorsReaderOrig, fieldsReaderOrig, cfsReader, simpleNormsProducer); + IOUtils.close(termVectorsLocal, fieldsReaderLocal, docValuesLocal, normsLocal, fields, dvProducer, + termVectorsReaderOrig, fieldsReaderOrig, cfsReader, normsProducer); notifyCoreClosedListeners(); } } diff --git a/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java b/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java index d5a1ff2e464..df335e28704 100644 --- a/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java +++ b/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java @@ -705,8 +705,7 @@ public class TestDemoDocValue extends LuceneTestCase { Directory directory = newDirectory(); // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1 IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); - // TODO: Fix the CFS/suffixing of Lucene41DocValues so it actually works with this - final DocValuesFormat fast = DocValuesFormat.forName("Memory"); + final DocValuesFormat fast = DocValuesFormat.forName("Lucene41"); final DocValuesFormat slow = DocValuesFormat.forName("SimpleText"); iwc.setCodec(new Lucene41Codec() { @Override diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java index 81c33706f49..0e5cdfea75b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java @@ -31,12 +31,12 @@ import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.asserting.AssertingPostingsFormat; import org.apache.lucene.codecs.lucene41.Lucene41Codec; +import org.apache.lucene.codecs.lucene41.Lucene41DocValuesFormat; import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat; import org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds; import org.apache.lucene.codecs.bloom.TestBloomFilteredLucene41Postings; import org.apache.lucene.codecs.diskdv.DiskDocValuesFormat; import org.apache.lucene.codecs.memory.DirectPostingsFormat; -import org.apache.lucene.codecs.memory.MemoryDocValuesFormat; import org.apache.lucene.codecs.memory.MemoryPostingsFormat; import org.apache.lucene.codecs.mockintblock.MockFixedIntBlockPostingsFormat; import org.apache.lucene.codecs.mockintblock.MockVariableIntBlockPostingsFormat; @@ -142,9 +142,9 @@ public class RandomCodec extends Lucene41Codec { new MemoryPostingsFormat(false, random.nextFloat())); addDocValues(avoidCodecs, + new Lucene41DocValuesFormat(), new DiskDocValuesFormat(), - new SimpleTextDocValuesFormat(), - new MemoryDocValuesFormat()); + new SimpleTextDocValuesFormat()); Collections.shuffle(formats, random); Collections.shuffle(dvFormats, random);