Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added on-disk / in memory impls for lucene 4.1 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1411240 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Simon Willnauer 2012-11-19 15:21:34 +00:00
parent 178c07f4e4
commit a3853a1129
39 changed files with 855 additions and 510 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
View File

@ -161,7 +161,7 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat {
IndexInput bloomIn = null;
boolean success = false;
try {
bloomIn = state.dir.openInput(bloomFileName, state.context);
bloomIn = state.directory.openInput(bloomFileName, state.context);
CodecUtil.checkHeader(bloomIn, BLOOM_CODEC_NAME, BLOOM_CODEC_VERSION,
BLOOM_CODEC_VERSION);
// // Load the hash function used in the BloomFilter

2
lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
View File

@ -842,7 +842,7 @@ public final class MemoryPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
final IndexInput in = state.dir.openInput(fileName, IOContext.READONCE);
final IndexInput in = state.directory.openInput(fileName, IOContext.READONCE);
final SortedMap<String,TermsReader> fields = new TreeMap<String,TermsReader>();

2
lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java
View File

@ -100,7 +100,7 @@ public abstract class PulsingPostingsFormat extends PostingsFormat {
docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state);
pulsingReader = new PulsingPostingsReader(docsReader);
FieldsProducer ret = new BlockTreeTermsReader(
state.dir, state.fieldInfos, state.segmentInfo,
state.directory, state.fieldInfos, state.segmentInfo,
pulsingReader,
state.context,
state.segmentSuffix,

2
lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepDocValuesProducer.java
View File

@ -45,7 +45,7 @@ public class SepDocValuesProducer extends PerDocProducerBase {
* {@link DocValues} instances for this segment and codec.
*/
public SepDocValuesProducer(SegmentReadState state) throws IOException {
docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.getDocCount(), state.dir, state.context);
docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.getDocCount(), state.directory, state.context);
}
@Override

2
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
View File

@ -67,7 +67,7 @@ class SimpleTextFieldsReader extends FieldsProducer {
final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD;
public SimpleTextFieldsReader(SegmentReadState state) throws IOException {
in = state.dir.openInput(SimpleTextPostingsFormat.getPostingsFileName(state.segmentInfo.name, state.segmentSuffix), state.context);
in = state.directory.openInput(SimpleTextPostingsFormat.getPostingsFileName(state.segmentInfo.name, state.segmentSuffix), state.context);
fieldInfos = state.fieldInfos;
fields = readFields(in.clone());

2
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java
View File

@ -68,7 +68,7 @@ public class SimpleTextPerDocProducer extends PerDocProducerBase {
this.segmentSuffix = segmentSuffix;
if (anyDocValuesFields(state.fieldInfos)) {
docValues = load(state.fieldInfos, state.segmentInfo.name,
state.segmentInfo.getDocCount(), state.dir, state.context);
state.segmentInfo.getDocCount(), state.directory, state.context);
} else {
docValues = new TreeMap<String, DocValues>();
}

2
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
View File

@ -82,7 +82,7 @@ public class SimpleTextSimpleDocValuesFormat extends SimpleDocValuesFormat {
@Override
public SimpleDVProducer fieldsProducer(SegmentReadState state) throws IOException {
return new SimpleTextDocValuesReader(state.fieldInfos, state.dir, state.segmentInfo, state.context);
return new SimpleTextDocValuesReader(state.fieldInfos, state.directory, state.segmentInfo, state.context);
}
/** the .dat file contains the data.

3
lucene/core/src/java/org/apache/lucene/codecs/SortedDocValuesConsumer.java
View File

@ -22,8 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.DocValues.SortedSource;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.Bits;
@ -31,6 +29,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.PriorityQueue;
//nocommit - this needs an abort() method? to free opened files?
public abstract class SortedDocValuesConsumer {
/** This is called, in value sort order, once per unique

2
lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesProducer.java
View File

@ -53,7 +53,7 @@ public class Lucene40DocValuesProducer extends PerDocProducerBase {
*/
public Lucene40DocValuesProducer(SegmentReadState state, String segmentSuffix) throws IOException {
if (anyDocValuesFields(state.fieldInfos)) {
cfs = new CompoundFileDirectory(state.dir,
cfs = new CompoundFileDirectory(state.directory,
IndexFileNames.segmentFileName(state.segmentInfo.name,
segmentSuffix, IndexFileNames.COMPOUND_FILE_EXTENSION),
state.context, false);

2
lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsBaseFormat.java
View File

@ -42,7 +42,7 @@ public final class Lucene40PostingsBaseFormat extends PostingsBaseFormat {
@Override
public PostingsReaderBase postingsReaderBase(SegmentReadState state) throws IOException {
return new Lucene40PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
return new Lucene40PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
}
@Override

4
lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java
View File

@ -248,12 +248,12 @@ public class Lucene40PostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postings = new Lucene40PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
PostingsReaderBase postings = new Lucene40PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
boolean success = false;
try {
FieldsProducer ret = new BlockTreeTermsReader(
state.dir,
state.directory,
state.fieldInfos,
state.segmentInfo,
postings,

8
lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java
View File

@ -35,7 +35,7 @@ import org.apache.lucene.codecs.lucene40.Lucene40NormsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
import org.apache.lucene.codecs.lucene41.values.Lucene41SimpleDocValuesFormat;
import org.apache.lucene.codecs.lucene41.values.Lucene41DocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
/**
@ -120,13 +120,11 @@ public class Lucene41Codec extends Codec {
return defaultFormat;
}
private final SimpleDocValuesFormat simpleDocValuesFormat = new Lucene41SimpleDocValuesFormat();
private final SimpleDocValuesFormat simpleDocValuesFormat = new Lucene41DocValuesFormat();
@Override
public SimpleDocValuesFormat simpleDocValuesFormat() {
// nocommit fixme
//return simpleDocValuesFormat;
return null;
return simpleDocValuesFormat ;
}
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");

2
lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsBaseFormat.java
View File

@ -41,7 +41,7 @@ public final class Lucene41PostingsBaseFormat extends PostingsBaseFormat {
@Override
public PostingsReaderBase postingsReaderBase(SegmentReadState state) throws IOException {
return new Lucene41PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
return new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
}
@Override

4
lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java
View File

@ -427,14 +427,14 @@ public final class Lucene41PostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.dir,
PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory,
state.fieldInfos,
state.segmentInfo,
state.context,
state.segmentSuffix);
boolean success = false;
try {
FieldsProducer ret = new BlockTreeTermsReader(state.dir,
FieldsProducer ret = new BlockTreeTermsReader(state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,

165
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValues.java Normal file
View File

@ -0,0 +1,165 @@
package org.apache.lucene.codecs.lucene41.values;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.lucene41.values.Lucene41DocValuesProducer.DocValuesFactory;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.packed.PackedInts;
import static org.apache.lucene.codecs.lucene41.values.Lucene41BinaryDocValuesConsumer.*;
public class Lucene41BinaryDocValues extends BinaryDocValues {
private final PackedInts.Reader index;
private final IndexInput data;
private final long baseOffset;
private final int size;
private int maxLength;
public Lucene41BinaryDocValues(IndexInput dataIn, long dataOffset, int size,
int maxLength, PackedInts.Reader index) throws IOException {
this.data = dataIn;
this.size = size;
this.maxLength = maxLength;
this.baseOffset = dataOffset;
this.index = index;
}
public void get(int docId, BytesRef result) {
try {
final long offset;
final int length;
if (index == null) {
offset = size * ((long) docId);
length = size;
} else {
offset = index.get(docId);
data.seek(baseOffset + offset);
// Safe to do 1+docID because we write sentinel at the end:
final long nextOffset = index.get(1 + docId);
length = (int) (nextOffset - offset);
}
result.offset = 0;
result.grow(length);
data.readBytes(result.bytes, 0, length);
result.length = length;
} catch (IOException ex) {
throw new IllegalStateException(
"failed to get value for docID: " + docId, ex);
}
}
@Override
public int size() {
return size;
}
@Override
public boolean isFixedLength() {
return index == null;
}
@Override
public int maxLength() {
return maxLength;
}
public static final class Factory extends DocValuesFactory<BinaryDocValues> {
private final IndexInput datIn;
private final IndexInput indexIn;
private final PackedInts.Header indexHeader;
private int size;
private int maxLength;
private long baseOffset;
private final int valueCount;
public Factory(Directory dir,
SegmentInfo segmentInfo, FieldInfo field, IOContext context
) throws IOException {
boolean success = false;
this.valueCount = segmentInfo.getDocCount();
IndexInput datIn = null;
IndexInput indexIn = null;
try {
datIn = dir.openInput(Lucene41DocValuesConsumer.getDocValuesFileName(
segmentInfo, field, Lucene41DocValuesConsumer.DATA_EXTENSION),
context);
CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START);
this.size = datIn.readInt();
this.maxLength = datIn.readInt();
this.baseOffset = datIn.getFilePointer();
if (size == VALUE_SIZE_VAR) {
indexIn = dir.openInput(Lucene41DocValuesConsumer
.getDocValuesFileName(segmentInfo, field,
Lucene41DocValuesConsumer.INDEX_EXTENSION), context);
CodecUtil.checkHeader(indexIn, CODEC_NAME, VERSION_START,
VERSION_START);
indexHeader = PackedInts.readHeader(indexIn);
} else {
indexIn = null;
indexHeader = null;
}
this.indexIn = indexIn;
this.datIn = datIn;
success = true;
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(datIn, indexIn);
}
}
}
public BinaryDocValues getDirect() throws IOException {
return new Lucene41BinaryDocValues(datIn.clone(), this.baseOffset, size,
maxLength,
indexHeader == null ? null : PackedInts.getDirectReaderNoHeader(
indexIn.clone(), indexHeader));
}
public BinaryDocValues getInMemory() throws IOException {
// nocommit simple in memory impl
PackedInts.Reader indexReader = indexHeader == null ? null : PackedInts
.getReaderNoHeader(indexIn.clone(), indexHeader);
PagedBytes bytes = new PagedBytes(15);
bytes.copy(datIn.clone(), indexReader == null ? size * valueCount
: indexReader.get(indexReader.size() - 1));
bytes.freeze(true);
return new Lucene41BinaryDocValues(bytes.getDataInput(), 0, size,
maxLength, indexReader);
}
@Override
public void close() throws IOException {
IOUtils.close(datIn, indexIn);
}
}
}

3
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesConsumer.java
View File

@ -45,14 +45,15 @@ public class Lucene41BinaryDocValuesConsumer extends BinaryDocValuesConsumer {
this.indexOut = indexOut;
CodecUtil.writeHeader(dataOut, CODEC_NAME, VERSION_START);
dataOut.writeInt(fixedLength ? maxLength: VALUE_SIZE_VAR);
dataOut.writeInt(maxLength);
CodecUtil.writeHeader(indexOut, CODEC_NAME, VERSION_START);
buffer = fixedLength ? null : new AppendingLongBuffer();
}
@Override
public void finish() throws IOException {
try {
indexOut.writeLong(offset);
if (buffer != null) {
Iterator iterator = buffer.iterator();
PackedInts.Writer writer = PackedInts.getWriter(indexOut,

216
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesProducer.java
View File

@ -1,216 +0,0 @@
package org.apache.lucene.codecs.lucene41.values;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.packed.PackedInts;
public class Lucene41BinaryDocValuesProducer extends DocValues {
private final IndexInput dataIn;
private final IndexInput indexIn;
private final int valueLength;
private final long dataFilePointer;
private long bytesToRead;
public Lucene41BinaryDocValuesProducer(IndexInput dataIn, IndexInput indexIn)
throws IOException {
this.dataIn = dataIn;
CodecUtil.checkHeader(dataIn, Lucene41BinaryDocValuesConsumer.CODEC_NAME,
Lucene41BinaryDocValuesConsumer.VERSION_START,
Lucene41BinaryDocValuesConsumer.VERSION_START);
valueLength = dataIn.readInt();
dataFilePointer = dataIn.getFilePointer();
CodecUtil.checkHeader(indexIn, Lucene41BinaryDocValuesConsumer.CODEC_NAME,
Lucene41BinaryDocValuesConsumer.VERSION_START,
Lucene41BinaryDocValuesConsumer.VERSION_START);
bytesToRead = indexIn.readLong();
if (valueLength == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR) {
this.indexIn = indexIn;
} else {
indexIn.close();
this.indexIn = null;
}
}
@Override
protected Source loadSource() throws IOException {
if (valueLength == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR) {
assert indexIn != null;
return new VarStraightSource(dataIn.clone(), indexIn.clone(), bytesToRead);
} else {
assert indexIn == null;
return new FixedStraightSource(dataIn.clone(), valueLength, bytesToRead);
}
}
@Override
protected Source loadDirectSource() throws IOException {
if (valueLength == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR) {
assert indexIn != null;
return new DirectVarStraightSource(dataIn.clone(), indexIn.clone(),
dataFilePointer);
} else {
assert indexIn == null;
return new DirectFixedStraightSource(dataIn.clone(), valueLength,
dataFilePointer);
}
}
@Override
public Type getType() {
return valueLength == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR ? Type.BYTES_VAR_STRAIGHT
: Type.BYTES_FIXED_STRAIGHT;
}
@Override
public void close() throws IOException {
super.close();
IOUtils.close(dataIn, indexIn);
}
static abstract class BytesSourceBase extends Source {
private final PagedBytes pagedBytes;
protected final IndexInput datIn;
protected final IndexInput idxIn;
protected final static int PAGED_BYTES_BITS = 15;
protected final PagedBytes.Reader data;
protected final long totalLengthInBytes;
protected BytesSourceBase(IndexInput datIn, IndexInput idxIn,
PagedBytes pagedBytes, long bytesToRead, Type type) throws IOException {
super(type);
assert bytesToRead <= datIn.length() : " file size is less than the expected size diff: "
+ (bytesToRead - datIn.length()) + " pos: " + datIn.getFilePointer();
this.datIn = datIn;
this.totalLengthInBytes = bytesToRead;
this.pagedBytes = pagedBytes;
this.pagedBytes.copy(datIn, bytesToRead);
data = pagedBytes.freeze(true);
this.idxIn = idxIn;
}
}
public final static class DirectVarStraightSource extends Source {
private final PackedInts.Reader index;
private final IndexInput data;
private final long baseOffset;
DirectVarStraightSource(IndexInput data, IndexInput index,
long dataFilePointer) throws IOException {
super(Type.BYTES_VAR_STRAIGHT);
this.data = data;
baseOffset = dataFilePointer;
this.index = PackedInts.getDirectReader(index); // nocommit read without
// header
}
private final int position(int docID) throws IOException {
final long offset = index.get(docID);
data.seek(baseOffset + offset);
// Safe to do 1+docID because we write sentinel at the end:
final long nextOffset = index.get(1 + docID);
return (int) (nextOffset - offset);
}
@Override
public BytesRef getBytes(int docID, BytesRef ref) {
try {
final int sizeToRead = position(docID);
ref.offset = 0;
ref.grow(sizeToRead);
data.readBytes(ref.bytes, 0, sizeToRead);
ref.length = sizeToRead;
return ref;
} catch (IOException ex) {
throw new IllegalStateException("failed to get value for docID: "
+ docID, ex);
}
}
}
private static final class VarStraightSource extends BytesSourceBase {
private final PackedInts.Reader addresses;
public VarStraightSource(IndexInput datIn, IndexInput idxIn,
long bytesToRead) throws IOException {
super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), bytesToRead,
Type.BYTES_VAR_STRAIGHT);
addresses = PackedInts.getReader(idxIn); // nocommit read without header
}
@Override
public BytesRef getBytes(int docID, BytesRef bytesRef) {
final long address = addresses.get(docID);
return data.fillSlice(bytesRef, address,
(int) (addresses.get(docID + 1) - address));
}
}
private final static class FixedStraightSource extends BytesSourceBase {
private final int size;
public FixedStraightSource(IndexInput datIn, int size, long bytesToRead)
throws IOException {
super(datIn, null, new PagedBytes(PAGED_BYTES_BITS), bytesToRead,
Type.BYTES_FIXED_STRAIGHT);
this.size = size;
}
@Override
public BytesRef getBytes(int docID, BytesRef bytesRef) {
return data.fillSlice(bytesRef, size * ((long) docID), size);
}
}
public final static class DirectFixedStraightSource extends Source {
private final int size;
private IndexInput data;
private long baseOffset;
DirectFixedStraightSource(IndexInput input, int size, long dataFilePointer) {
super(Type.BYTES_FIXED_STRAIGHT);
this.size = size;
this.data = input;
baseOffset = dataFilePointer;
}
@Override
public BytesRef getBytes(int docID, BytesRef ref) {
try {
data.seek(baseOffset + size * ((long) docID));
ref.offset = 0;
ref.grow(size);
data.readBytes(ref.bytes, 0, size);
ref.length = size;
return ref;
} catch (IOException ex) {
throw new IllegalStateException("failed to get value for docID: "
+ docID, ex);
}
}
}
}

64
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java
View File

@ -25,6 +25,7 @@ import org.apache.lucene.codecs.SortedDocValuesConsumer;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@ -36,15 +37,18 @@ public class Lucene41DocValuesConsumer extends SimpleDVConsumer {
*/
static final String INDEX_EXTENSION = "idx";
static final String OFFSET_EXTENSION = "off";
/**
* Filename extension for data files.
*/
static final String DATA_EXTENSION = "dat";
static final String DV_SEGMENT_SUFFIX = "dv";
static final String DV_SEGMENT_SUFFIX = "sdv"; // nocommit change to dv
private final SegmentInfo info;
private final Directory dir;
private Directory cfs;
private final IOContext context;
Lucene41DocValuesConsumer(Directory dir, SegmentInfo si, IOContext context)
@ -54,18 +58,28 @@ public class Lucene41DocValuesConsumer extends SimpleDVConsumer {
this.context = context;
}
private synchronized Directory getDirectory() throws IOException {
if (cfs == null) {
cfs = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(info.name, DV_SEGMENT_SUFFIX,
IndexFileNames.COMPOUND_FILE_EXTENSION), context, true);
}
return cfs;
}
@Override
public void close() throws IOException {}
public void close() throws IOException {
IOUtils.close(cfs);
}
@Override
public NumericDocValuesConsumer addNumericField(FieldInfo field,
long minValue, long maxValue) throws IOException {
String name = IndexFileNames.segmentFileName(this.info.name + "_"
+ field.number, DV_SEGMENT_SUFFIX, DATA_EXTENSION);
String name = getDocValuesFileName(info, field, DATA_EXTENSION);
IndexOutput dataOut = null;
boolean success = false;
try {
dataOut = dir.createOutput(name, context);
dataOut = getDirectory().createOutput(name, context);
Lucene41NumericDocValuesConsumer consumer = new Lucene41NumericDocValuesConsumer(
dataOut, minValue, maxValue, info.getDocCount());
success = true;
@ -80,16 +94,14 @@ public class Lucene41DocValuesConsumer extends SimpleDVConsumer {
@Override
public BinaryDocValuesConsumer addBinaryField(FieldInfo field,
boolean fixedLength, int maxLength) throws IOException {
String nameData = IndexFileNames.segmentFileName(this.info.name + "_"
+ field.number, DV_SEGMENT_SUFFIX, DATA_EXTENSION);
String idxOut = IndexFileNames.segmentFileName(this.info.name + "_"
+ field.number, DV_SEGMENT_SUFFIX, INDEX_EXTENSION);
String nameData = getDocValuesFileName(info, field, DATA_EXTENSION);
String idxOut = getDocValuesFileName(info, field, INDEX_EXTENSION);
boolean success = false;
IndexOutput dataOut = null;
IndexOutput indexOut = null;
try {
dataOut = dir.createOutput(nameData, context);
indexOut = dir.createOutput(idxOut, context);
dataOut = getDirectory().createOutput(nameData, context);
indexOut = getDirectory().createOutput(idxOut, context);
Lucene41BinaryDocValuesConsumer consumer = new Lucene41BinaryDocValuesConsumer(
dataOut, indexOut, fixedLength, maxLength);
success = true;
@ -101,11 +113,39 @@ public class Lucene41DocValuesConsumer extends SimpleDVConsumer {
}
}
static String getDocValuesFileName(SegmentInfo info, FieldInfo field, String extension) {
return IndexFileNames.segmentFileName(info.name + "_"
+ field.number, DV_SEGMENT_SUFFIX, extension);
}
@Override
public SortedDocValuesConsumer addSortedField(FieldInfo field,
int valueCount, boolean fixedLength, int maxLength)
throws IOException {
return null;
String nameData = getDocValuesFileName(info, field, DATA_EXTENSION);
String idxOut = getDocValuesFileName(info, field, INDEX_EXTENSION);
String offOut = getDocValuesFileName(info, field, OFFSET_EXTENSION);
boolean success = false;
IndexOutput dataOut = null;
IndexOutput indexOut = null;
IndexOutput offsetOut = null;
try {
dataOut = getDirectory().createOutput(nameData, context);
indexOut = getDirectory().createOutput(idxOut, context);
if (fixedLength) {
offsetOut = null;
} else {
offsetOut = getDirectory().createOutput(offOut, context);
}
Lucene41SortedDocValuesConsumer consumer = new Lucene41SortedDocValuesConsumer(
dataOut, indexOut, offsetOut, valueCount, maxLength);
success = true;
return consumer;
} finally {
if (!success) {
IOUtils.close(dataOut, indexOut);
}
}
}
}

42
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesFormat.java Normal file
View File

@ -0,0 +1,42 @@
package org.apache.lucene.codecs.lucene41.values;
import java.io.IOException;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SimpleDVProducer;
import org.apache.lucene.codecs.SimpleDocValuesFormat;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class Lucene41DocValuesFormat extends SimpleDocValuesFormat {
@Override
public SimpleDVConsumer fieldsConsumer(SegmentWriteState state)
throws IOException {
return new Lucene41DocValuesConsumer(state.directory, state.segmentInfo, state.context);
}
@Override
public SimpleDVProducer fieldsProducer(SegmentReadState state)
throws IOException {
return new Lucene41DocValuesProducer(state.directory, state.segmentInfo, state.context);
}
}

114
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java Normal file
View File

@ -0,0 +1,114 @@
package org.apache.lucene.codecs.lucene41.values;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.codecs.SimpleDVProducer;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.IOUtils;
// nocommit
public class Lucene41DocValuesProducer extends SimpleDVProducer {
private final CompoundFileDirectory cfs;
private IOContext context;
private final SegmentInfo info;
private final Map<String,DocValuesFactory<NumericDocValues>> numeric = new HashMap<String,DocValuesFactory<NumericDocValues>>();
private final Map<String,DocValuesFactory<BinaryDocValues>> binary = new HashMap<String,DocValuesFactory<BinaryDocValues>>();
private final Map<String,DocValuesFactory<SortedDocValues>> sorted = new HashMap<String,DocValuesFactory<SortedDocValues>>();
public Lucene41DocValuesProducer(Directory dir, SegmentInfo segmentInfo,
IOContext context) throws IOException {
this.cfs = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(
segmentInfo.name, Lucene41DocValuesConsumer.DV_SEGMENT_SUFFIX,
IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
this.context = context;
this.info = segmentInfo;
}
@Override
public void close() throws IOException {
try {
List<Closeable> closeables = new ArrayList<Closeable>(numeric.values());
closeables.addAll(binary.values());
closeables.addAll(sorted.values());
IOUtils.close(closeables);
} finally {
IOUtils.close(cfs);
}
}
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
//nocommit do we need to sync that?
DocValuesFactory<NumericDocValues> docValuesFactory = numeric
.get(field.name);
if (docValuesFactory == null) {
numeric.put(field.name,
docValuesFactory = new Lucene41NumericDocValues.Factory(this.cfs,
this.info, field, context));
}
return docValuesFactory.getDirect();
}
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
//nocommit do we need to sync that?
DocValuesFactory<BinaryDocValues> docValuesFactory = binary.get(field.name);
if (docValuesFactory == null) {
binary.put(field.name,
docValuesFactory = new Lucene41BinaryDocValues.Factory(this.cfs,
this.info, field, context));
}
return docValuesFactory.getDirect();
}
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
//nocommit do we need to sync that?
DocValuesFactory<SortedDocValues> docValuesFactory = sorted.get(field.name);
if (docValuesFactory == null) {
sorted.put(field.name,
docValuesFactory = new Lucene41SortedDocValues.Factory(this.cfs,
this.info, field, context));
}
return docValuesFactory.getDirect();
}
public static abstract class DocValuesFactory<T> implements Closeable {
public abstract T getDirect() throws IOException;
public abstract T getInMemory() throws IOException;
}
}

107
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValues.java Normal file
View File

@ -0,0 +1,107 @@
package org.apache.lucene.codecs.lucene41.values;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.lucene41.values.Lucene41DocValuesProducer.DocValuesFactory;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedInts.Reader;
public class Lucene41NumericDocValues extends NumericDocValues {
private final long minValue;
private final Reader values;
private final long maxValue;
public Lucene41NumericDocValues(PackedInts.Reader reader, long minValue, long maxValue) {
this.values = reader;
this.minValue = minValue;
this.maxValue = maxValue;
}
@Override
public long get(int docID) {
assert docID >= 0;
return values.get(docID) + minValue;
}
public static final class Factory extends DocValuesFactory<NumericDocValues> {
private final IndexInput datIn;
private final PackedInts.Header header;
private final long minValue;
private final long maxValue;
public Factory(Directory dir, SegmentInfo segmentInfo, FieldInfo field, IOContext context) throws IOException {
this.datIn = dir.openInput(Lucene41DocValuesConsumer.getDocValuesFileName(segmentInfo, field,
Lucene41DocValuesConsumer.DATA_EXTENSION), context);
boolean success = false;
try {
CodecUtil.checkHeader(datIn, Lucene41NumericDocValuesConsumer.CODEC_NAME,
Lucene41NumericDocValuesConsumer.VERSION_START,
Lucene41NumericDocValuesConsumer.VERSION_START);
minValue = datIn.readLong();
maxValue = datIn.readLong();
this.header = PackedInts.readHeader(datIn);
success = true;
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(datIn);
}
}
}
public NumericDocValues getDirect() throws IOException {
IndexInput input = datIn.clone();
return new Lucene41NumericDocValues(PackedInts.getDirectReaderNoHeader(input, header), minValue, maxValue);
}
public NumericDocValues getInMemory() throws IOException {
IndexInput input = datIn.clone();
return new Lucene41NumericDocValues(PackedInts.getReaderNoHeader(input, header), minValue, maxValue);
}
@Override
public void close() throws IOException {
IOUtils.close(datIn);
}
}
@Override
public long minValue() {
return minValue;
}
@Override
public long maxValue() {
return maxValue;
}
@Override
public int size() {
return values.size();
}
}

3
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesConsumer.java
View File

@ -22,12 +22,11 @@ public class Lucene41NumericDocValuesConsumer extends NumericDocValuesConsumer {
final int bitsRequired = delta < 0 ? 64 : PackedInts.bitsRequired(delta);
CodecUtil.writeHeader(output, CODEC_NAME, VERSION_START);
output.writeLong(minValue);
output.writeLong(maxValue);
this.minValue = minValue;
// nocommit write without header?
this.writer = PackedInts.getWriter(output, valueCount, bitsRequired, PackedInts.FASTEST);
}
@Override
public void add(long value) throws IOException {
writer.add(value-minValue);

115
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesProducer.java
View File

@ -1,115 +0,0 @@
package org.apache.lucene.codecs.lucene41.values;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesArraySource;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.PackedInts;
public class Lucene41NumericDocValuesProducer extends DocValues {
private final IndexInput datIn;
private final long minValue;
public Lucene41NumericDocValuesProducer(IndexInput input, int numDocs)
throws IOException {
datIn = input;
boolean success = false;
try {
CodecUtil.checkHeader(datIn, Lucene41NumericDocValuesConsumer.CODEC_NAME,
Lucene41NumericDocValuesConsumer.VERSION_START,
Lucene41NumericDocValuesConsumer.VERSION_START);
minValue = datIn.readLong();
success = true;
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(datIn);
}
}
}
/**
* Loads the actual values. You may call this more than once, eg if you
* already previously loaded but then discarded the Source.
*/
@Override
protected Source loadSource() throws IOException {
boolean success = false;
final Source source;
IndexInput input = null;
try {
input = datIn.clone();
source = new PackedIntsSource(input, false, minValue);
success = true;
return source;
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(input, datIn);
}
}
}
@Override
public void close() throws IOException {
super.close();
datIn.close();
}
@Override
public Type getType() {
return Type.VAR_INTS;
}
@Override
protected Source loadDirectSource() throws IOException {
return new PackedIntsSource(datIn.clone(), true, minValue);
}
static class PackedIntsSource extends Source {
private final PackedInts.Reader values;
private final long minValue;
public PackedIntsSource(IndexInput dataIn, boolean direct, long minValue)
throws IOException {
super(Type.VAR_INTS);
this.minValue = minValue;
values = direct ? PackedInts.getDirectReader(dataIn) : PackedInts
.getReader(dataIn);
}
@Override
public BytesRef getBytes(int docID, BytesRef ref) {
ref.grow(8);
DocValuesArraySource.copyLong(ref, getInt(docID));
return ref;
}
@Override
public long getInt(int docID) {
assert docID >= 0;
return values.get(docID) + minValue;
}
}
}

113
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SimpleDocValuesFormat.java
View File

@ -1,113 +0,0 @@
package org.apache.lucene.codecs.lucene41.values;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.Closeable;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.TreeMap;
import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.PerDocProducerBase;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SimpleDVProducer;
import org.apache.lucene.codecs.SimpleDocValuesFormat;
import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.IOUtils;
public class Lucene41SimpleDocValuesFormat extends SimpleDocValuesFormat {
@Override
public SimpleDVConsumer fieldsConsumer(SegmentWriteState state)
throws IOException {
return new Lucene41DocValuesConsumer(state.directory, state.segmentInfo, state.context);
}
@Override
public SimpleDVProducer fieldsProducer(SegmentReadState state)
throws IOException {
// nocommit fixme
// return new Lucene41PerdocProducer(state);
return null;
}
//nocommit this is equivalent to sep - we should pack in CFS
private static final class Lucene41DocValuesReader extends PerDocProducerBase {
private final TreeMap<String, DocValues> docValues;
/**
* Creates a new {@link Lucene41DocValuesReader} instance and loads all
* {@link DocValues} instances for this segment and codec.
*/
public Lucene41DocValuesReader(SegmentReadState state) throws IOException {
docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.getDocCount(), state.dir, state.context);
}
@Override
protected Map<String,DocValues> docValues() {
return docValues;
}
@Override
protected void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
IOUtils.close(closeables);
}
@Override
protected DocValues loadDocValues(int docCount, Directory dir, String id,
Type type, IOContext context) throws IOException {
switch (type) {
case FIXED_INTS_16:
case FIXED_INTS_32:
case FIXED_INTS_64:
case FIXED_INTS_8:
case VAR_INTS:
case FLOAT_32:
case FLOAT_64:
return new Lucene41NumericDocValuesProducer(dir.openInput( IndexFileNames.segmentFileName(
id, Lucene41DocValuesConsumer.DV_SEGMENT_SUFFIX, Lucene41DocValuesConsumer.DATA_EXTENSION), context), docCount);
case BYTES_FIXED_STRAIGHT:
case BYTES_FIXED_DEREF:
case BYTES_VAR_STRAIGHT:
case BYTES_VAR_DEREF:
//nocommit cose in case of an exception
IndexInput dataIn = dir.openInput(IndexFileNames.segmentFileName(id,
Lucene41DocValuesConsumer.DV_SEGMENT_SUFFIX,
Lucene41DocValuesConsumer.DATA_EXTENSION), context);
IndexInput indexIn = dir.openInput(IndexFileNames.segmentFileName(id,
Lucene41DocValuesConsumer.DV_SEGMENT_SUFFIX,
Lucene41DocValuesConsumer.INDEX_EXTENSION), context);
return new Lucene41BinaryDocValuesProducer(dataIn, indexIn);
case BYTES_VAR_SORTED:
case BYTES_FIXED_SORTED:
default:
throw new IllegalStateException("unrecognized index values mode " + type);
}
}
}
}

193
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java Normal file
View File

@ -0,0 +1,193 @@
package org.apache.lucene.codecs.lucene41.values;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.lucene41.values.Lucene41DocValuesProducer.DocValuesFactory;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.packed.PackedInts;
import static org.apache.lucene.codecs.lucene41.values.Lucene41SortedDocValuesConsumer.*;
public class Lucene41SortedDocValues extends SortedDocValues {
private final PackedInts.Reader docToOrdIndex;
private final PackedInts.Reader ordToOffsetIndex;
private final IndexInput data;
private final long baseOffset;
private final int valueCount;
private int size;
private int maxLength;
public Lucene41SortedDocValues(IndexInput dataIn, long dataOffset, int size,
int maxLength, PackedInts.Reader index, PackedInts.Reader offsets)
throws IOException {
this.data = dataIn;
this.size = size;
this.maxLength = maxLength;
this.baseOffset = dataOffset;
this.docToOrdIndex = index;
this.valueCount = docToOrdIndex.size();
ordToOffsetIndex = offsets;
}
@Override
public int getOrd(int docID) {
return (int) docToOrdIndex.get(docID);
}
@Override
public void lookupOrd(int ord, BytesRef result) {
try {
final long offset;
final int length;
if (ordToOffsetIndex != null) {
offset = ordToOffsetIndex.get(ord);
// 1+ord is safe because we write a sentinel at the end
final long nextOffset = ordToOffsetIndex.get(1 + ord);
length = (int) (nextOffset - offset);
} else {
length = size;
offset = size * ord;
}
data.seek(baseOffset + offset);
result.offset = 0;
result.grow(length);
data.readBytes(result.bytes, 0, length);
result.length = length;
} catch (IOException ex) {
throw new IllegalStateException("failed", ex);
}
}
@Override
public int getValueCount() {
return valueCount;
}
@Override
public int size() {
return size;
}
@Override
public boolean isFixedLength() {
return ordToOffsetIndex == null;
}
@Override
public int maxLength() {
return maxLength;
}
public static final class Factory extends
DocValuesFactory<SortedDocValues> {
private final IndexInput datIn;
private final IndexInput offsetIn;
private final PackedInts.Header offsetHeader;
private final IndexInput indexIn;
private final PackedInts.Header indexHeader;
private int size;
private int maxLength;
private long baseOffset;
public Factory(Directory dir,
SegmentInfo segmentInfo, FieldInfo field, IOContext context)
throws IOException {
boolean success = false;
IndexInput datIn = null;
IndexInput indexIn = null;
IndexInput offsetIn = null;
try {
datIn = dir.openInput(Lucene41DocValuesConsumer.getDocValuesFileName(
segmentInfo, field, Lucene41DocValuesConsumer.DATA_EXTENSION),
context);
CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START);
indexIn = dir.openInput(Lucene41DocValuesConsumer.getDocValuesFileName(
segmentInfo, field, Lucene41DocValuesConsumer.INDEX_EXTENSION),
context);
CodecUtil
.checkHeader(indexIn, CODEC_NAME, VERSION_START, VERSION_START);
indexHeader = PackedInts.readHeader(indexIn);
this.size = datIn.readInt();
this.maxLength = datIn.readInt();
this.baseOffset = datIn.getFilePointer();
if (size == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR) {
offsetIn = dir.openInput(Lucene41DocValuesConsumer
.getDocValuesFileName(segmentInfo, field,
Lucene41DocValuesConsumer.OFFSET_EXTENSION), context);
CodecUtil.checkHeader(offsetIn, CODEC_NAME, VERSION_START,
VERSION_START);
this.offsetHeader = PackedInts.readHeader(offsetIn);
} else {
offsetIn = null;
this.offsetHeader = null;
}
this.offsetIn = offsetIn;
this.indexIn = indexIn;
this.datIn = datIn;
success = true;
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(datIn, indexIn, offsetIn);
}
}
}
public SortedDocValues getDirect() throws IOException {
return new Lucene41SortedDocValues(datIn.clone(), this.baseOffset, size,
maxLength, PackedInts.getDirectReaderNoHeader(indexIn.clone(),
indexHeader), offsetHeader == null ? null
: PackedInts.getDirectReaderNoHeader(offsetIn.clone(),
offsetHeader));
}
public Lucene41SortedDocValues getInMemory() throws IOException {
// nocommit simple in memory impl
PackedInts.Reader offsetReader = offsetHeader == null ? null : PackedInts
.getReaderNoHeader(offsetIn.clone(), offsetHeader);
PackedInts.Reader indexReader = PackedInts.getReaderNoHeader(
indexIn.clone(), indexHeader);
PagedBytes bytes = new PagedBytes(15);
bytes.copy(
datIn.clone(),
offsetReader == null ? size * indexReader.size() : offsetReader
.get(offsetReader.size() - 1));
bytes.freeze(true);
return new Lucene41SortedDocValues(bytes.getDataInput(), 0, size,
maxLength, indexReader, offsetReader);
}
@Override
public void close() throws IOException {
IOUtils.close(datIn, indexIn, offsetIn);
}
}
}

74
lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValuesConsumer.java
View File

@ -18,18 +18,86 @@ package org.apache.lucene.codecs.lucene41.values;
*/
import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.SortedDocValuesConsumer;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.AppendingLongBuffer;
import org.apache.lucene.util.packed.AppendingLongBuffer.Iterator;
import org.apache.lucene.util.packed.PackedInts;
public class Lucene41SortedDocValuesConsumer extends SortedDocValuesConsumer {
static final int VERSION_START = -1;
static final String CODEC_NAME = "Lucene41Sorted";
static final int VALUE_SIZE_VAR = -1;
private final IndexOutput data;
private final IndexOutput index;
private final IndexOutput offsets;
private final AppendingLongBuffer offsetBuffer;
private final PackedInts.Writer ords;
private long offset = 0;
private final int valueCount;
public Lucene41SortedDocValuesConsumer(IndexOutput dataOut,
IndexOutput indexOut, IndexOutput offsetOut, int valueCount, int maxLength)
throws IOException {
int size;
if (offsetOut != null) {
size = VALUE_SIZE_VAR;
offsetBuffer = new AppendingLongBuffer();
this.offsets = offsetOut;
CodecUtil.writeHeader(offsetOut, CODEC_NAME, VERSION_START);
} else {
size = maxLength;
offsetBuffer = null;
this.offsets = null;
}
CodecUtil.writeHeader(dataOut, CODEC_NAME, VERSION_START);
dataOut.writeInt(size);
dataOut.writeInt(maxLength);
CodecUtil.writeHeader(indexOut, CODEC_NAME, VERSION_START);
this.data = dataOut;
this.index = indexOut;
this.valueCount = valueCount;
ords = PackedInts.getWriter(index, valueCount,
PackedInts.bitsRequired(valueCount-1), PackedInts.DEFAULT);
}
@Override
public void addValue(BytesRef value) throws IOException {}
public void addValue(BytesRef value) throws IOException {
data.writeBytes(value.bytes, value.offset, value.length);
if (offsetBuffer != null) {
offsetBuffer.add(offset);
offset += value.length;
}
}
@Override
public void addDoc(int ord) throws IOException {}
public void addDoc(int ord) throws IOException {
ords.add(ord);
}
@Override
public void finish() throws IOException {}
public void finish() throws IOException {
try {
ords.finish();
if (offsetBuffer != null) {
final int bitsRequired = PackedInts.bitsRequired(offset);
Iterator iterator = offsetBuffer.iterator();
PackedInts.Writer writer = PackedInts.getWriter(offsets, valueCount+1,
bitsRequired, PackedInts.DEFAULT);
while (iterator.hasNext()) {
writer.add(iterator.next());
}
writer.add(offset); // total # bytes
writer.finish();
}
} finally {
IOUtils.close(data, index, offsets);
}
}
}

6
lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java
View File

@ -28,7 +28,7 @@ import org.apache.lucene.store.IOContext;
*/
public class SegmentReadState {
/** {@link Directory} where this segment is read from. */
public final Directory dir;
public final Directory directory;
/** {@link SegmentInfo} describing this segment. */
public final SegmentInfo segmentInfo;
@ -73,7 +73,7 @@ public class SegmentReadState {
IOContext context,
int termsIndexDivisor,
String segmentSuffix) {
this.dir = dir;
this.directory = dir;
this.segmentInfo = info;
this.fieldInfos = fieldInfos;
this.context = context;
@ -84,7 +84,7 @@ public class SegmentReadState {
/** Create a {@code SegmentReadState}. */
public SegmentReadState(SegmentReadState other,
String newSegmentSuffix) {
this.dir = other.dir;
this.directory = other.directory;
this.segmentInfo = other.segmentInfo;
this.fieldInfos = other.fieldInfos;
this.context = other.context;

1
lucene/core/src/java/org/apache/lucene/index/SortedBytesDVWriter.java
View File

@ -141,6 +141,7 @@ class SortedBytesDVWriter {
}
iwBytesUsed.addAndGet(-sortedValueRamUsage);
reset();
consumer.finish();
}
public void abort() {

2
lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
View File

@ -219,7 +219,9 @@ final class CompoundFileWriter implements Closeable{
assert !seenIDs.contains(id): "file=\"" + name + "\" maps to id=\"" + id + "\", which was already written";
seenIDs.add(id);
final DirectCFSIndexOutput out;
if ((outputLocked = outputTaken.compareAndSet(false, true))) {
// System.out.println(entry.file);
out = new DirectCFSIndexOutput(getOutput(), entry, false);
} else {
entry.dir = this.directory;

25
lucene/core/src/java/org/apache/lucene/util/PagedBytes.java
View File

@ -405,12 +405,13 @@ public final class PagedBytes {
return pointer;
}
public final class PagedBytesDataInput extends DataInput {
public final class PagedBytesDataInput extends IndexInput {
private int currentBlockIndex;
private int currentBlockUpto;
private byte[] currentBlock;
PagedBytesDataInput() {
super("PagedBytesIndexInput");
currentBlock = blocks.get(0);
}
@ -471,6 +472,28 @@ public final class PagedBytes {
currentBlockUpto = 0;
currentBlock = blocks.get(currentBlockIndex);
}
@Override
public void close() throws IOException {
//
}
@Override
public long getFilePointer() {
return currentBlockIndex * blockSize + currentBlockUpto;
}
@Override
public void seek(long pos) throws IOException {
currentBlockIndex = (int) (pos >> blockBits);
currentBlock = blocks.get(currentBlockIndex);
currentBlockUpto = (int) (pos & blockMask);
}
@Override
public long length() {
return upto;
}
}
public final class PagedBytesDataOutput extends DataOutput {

37
lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
View File

@ -25,6 +25,7 @@ import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.packed.PackedInts.Format;
/**
* Simplistic compression for array of unsigned long values.
@ -775,6 +776,11 @@ public class PackedInts {
throw new AssertionError("Unknown Writer format: " + format);
}
}
//nocommit javadoc
public static Reader getReaderNoHeader(DataInput in, Header header) throws IOException {
return getReaderNoHeader(in, header.format, header.version, header.valueCount, header.bitsPerValue);
}
/**
* Restore a {@link Reader} from a stream.
@ -887,6 +893,11 @@ public class PackedInts {
throw new AssertionError("Unknwown format: " + format);
}
}
//nocommit javadoc
public static Reader getDirectReaderNoHeader(IndexInput in, Header header) throws IOException {
return getDirectReaderNoHeader(in, header.format, header.version, header.valueCount, header.bitsPerValue);
}
/**
* Construct a direct {@link Reader} from an {@link IndexInput}. This method
@ -1117,5 +1128,31 @@ public class PackedInts {
}
}
}
//nocommit javadoc
public static Header readHeader(IndexInput in) throws IOException {
final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
final int bitsPerValue = in.readVInt();
assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
final int valueCount = in.readVInt();
final Format format = Format.byId(in.readVInt());
return new Header(format, valueCount, bitsPerValue, version);
}
public static class Header {
private final Format format;
private final int valueCount;
private final int bitsPerValue;
private final int version;
public Header(Format format, int valueCount, int bitsPerValue, int version) {
this.format = format;
this.valueCount = valueCount;
this.bitsPerValue = bitsPerValue;
this.version = version;
}
}
}

2
lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java
View File

@ -50,7 +50,7 @@ import org.apache.lucene.util.LuceneTestCase;
* to this class.
*/
// nocommit don't suppress any:
@SuppressCodecs({"Asserting", "Direct", "Memory", "Lucene41", "MockRandom", "Lucene40", "Compressing"})
@SuppressCodecs({"Asserting", "Direct", "Memory", "MockRandom", "Lucene40", "Compressing"})
public class TestDemoDocValue extends LuceneTestCase {
public void testDemoNumber() throws IOException {

6
lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java
View File

@ -91,12 +91,12 @@ public final class Lucene41WithOrds extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postings = new Lucene41PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
PostingsReaderBase postings = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
TermsIndexReaderBase indexReader;
boolean success = false;
try {
indexReader = new FixedGapTermsIndexReader(state.dir,
indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
@ -112,7 +112,7 @@ public final class Lucene41WithOrds extends PostingsFormat {
success = false;
try {
FieldsProducer ret = new BlockTermsReader(indexReader,
state.dir,
state.directory,
state.fieldInfos,
state.segmentInfo,
postings,

6
lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java
View File

@ -156,7 +156,7 @@ public final class MockFixedIntBlockPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postingsReader = new SepPostingsReader(state.dir,
PostingsReaderBase postingsReader = new SepPostingsReader(state.directory,
state.fieldInfos,
state.segmentInfo,
state.context,
@ -165,7 +165,7 @@ public final class MockFixedIntBlockPostingsFormat extends PostingsFormat {
TermsIndexReaderBase indexReader;
boolean success = false;
try {
indexReader = new FixedGapTermsIndexReader(state.dir,
indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
@ -181,7 +181,7 @@ public final class MockFixedIntBlockPostingsFormat extends PostingsFormat {
success = false;
try {
FieldsProducer ret = new BlockTermsReader(indexReader,
state.dir,
state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,

6
lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java
View File

@ -180,7 +180,7 @@ public final class MockVariableIntBlockPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postingsReader = new SepPostingsReader(state.dir,
PostingsReaderBase postingsReader = new SepPostingsReader(state.directory,
state.fieldInfos,
state.segmentInfo,
state.context,
@ -189,7 +189,7 @@ public final class MockVariableIntBlockPostingsFormat extends PostingsFormat {
TermsIndexReaderBase indexReader;
boolean success = false;
try {
indexReader = new FixedGapTermsIndexReader(state.dir,
indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
@ -205,7 +205,7 @@ public final class MockVariableIntBlockPostingsFormat extends PostingsFormat {
success = false;
try {
FieldsProducer ret = new BlockTermsReader(indexReader,
state.dir,
state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,

14
lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
View File

@ -288,7 +288,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, SEED_EXT);
final IndexInput in = state.dir.openInput(seedFileName, state.context);
final IndexInput in = state.directory.openInput(seedFileName, state.context);
final long seed = in.readLong();
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " formatID=" + state.segmentSuffix + " seed=" + seed);
@ -308,13 +308,13 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading Sep postings");
}
postingsReader = new SepPostingsReader(state.dir, state.fieldInfos, state.segmentInfo,
postingsReader = new SepPostingsReader(state.directory, state.fieldInfos, state.segmentInfo,
state.context, new MockIntStreamFactory(random), state.segmentSuffix);
} else {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading Standard postings");
}
postingsReader = new Lucene41PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
postingsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
}
if (random.nextBoolean()) {
@ -335,7 +335,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
boolean success = false;
try {
fields = new BlockTreeTermsReader(state.dir,
fields = new BlockTreeTermsReader(state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,
@ -369,7 +369,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: fixed-gap terms index (divisor=" + state.termsIndexDivisor + ")");
}
indexReader = new FixedGapTermsIndexReader(state.dir,
indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
@ -385,7 +385,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: variable-gap terms index (divisor=" + state.termsIndexDivisor + ")");
}
indexReader = new VariableGapTermsIndexReader(state.dir,
indexReader = new VariableGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
@ -405,7 +405,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
success = false;
try {
fields = new BlockTermsReader(indexReader,
state.dir,
state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,

6
lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java
View File

@ -83,13 +83,13 @@ public final class MockSepPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, state.fieldInfos, state.segmentInfo,
PostingsReaderBase postingsReader = new SepPostingsReader(state.directory, state.fieldInfos, state.segmentInfo,
state.context, new MockSingleIntFactory(), state.segmentSuffix);
TermsIndexReaderBase indexReader;
boolean success = false;
try {
indexReader = new FixedGapTermsIndexReader(state.dir,
indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
@ -105,7 +105,7 @@ public final class MockSepPostingsFormat extends PostingsFormat {
success = false;
try {
FieldsProducer ret = new BlockTermsReader(indexReader,
state.dir,
state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,

4
lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java
View File

@ -77,11 +77,11 @@ public final class NestedPulsingPostingsFormat extends PostingsFormat {
PostingsReaderBase pulsingReader = null;
boolean success = false;
try {
docsReader = new Lucene41PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
docsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
pulsingReaderInner = new PulsingPostingsReader(docsReader);
pulsingReader = new PulsingPostingsReader(pulsingReaderInner);
FieldsProducer ret = new BlockTreeTermsReader(
state.dir, state.fieldInfos, state.segmentInfo,
state.directory, state.fieldInfos, state.segmentInfo,
pulsingReader,
state.context,
state.segmentSuffix,

2
lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
View File

@ -554,7 +554,7 @@ public final class RAMOnlyPostingsFormat extends PostingsFormat {
// Load our ID:
final String idFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, readState.segmentSuffix, ID_EXTENSION);
IndexInput in = readState.dir.openInput(idFileName, readState.context);
IndexInput in = readState.directory.openInput(idFileName, readState.context);
boolean success = false;
final int id;
try {