mirror of https://github.com/apache/lucene.git
LUCENE-9447: Make BEST_COMPRESSION better with highly compressible data. (#1762)
This makes BEST_COMPRESSION split blocks into sub blocks and use preset dictionaries to improve compression ratios.
This commit is contained in:
Adrien Grand
GitHub
parent
e6fa5a5fc4
commit
913976dbf7
|
|
@ -198,6 +198,9 @@ Optimizations
|
|||
* LUCENE-9395: ConstantValuesSource now shares a single DoubleValues
|
||||
instance across all segments (Tony Xu)
|
||||
|
||||
* LUCENE-9447: BEST_COMPRESSION now provides higher compression ratios on highly
|
||||
compressible data. (Adrien Grand)
|
||||
|
||||
* LUCENE-9373: FunctionMatchQuery now accepts a "matchCost" optimization hint.
|
||||
(Maxim Glazkov, David Smiley)
|
||||
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ import org.apache.lucene.util.packed.DirectMonotonicWriter;
|
|||
* larger than (<code>2<sup>31</sup> - 2<sup>14</sup></code>) bytes.
|
||||
* @lucene.experimental
|
||||
*/
|
||||
public final class Lucene50StoredFieldsFormat extends StoredFieldsFormat {
|
||||
public class Lucene50StoredFieldsFormat extends StoredFieldsFormat {
|
||||
|
||||
/** Configuration option for stored fields. */
|
||||
public static enum Mode {
|
||||
|
|
@ -126,7 +126,7 @@ public final class Lucene50StoredFieldsFormat extends StoredFieldsFormat {
|
|||
}
|
||||
|
||||
@Override
|
||||
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
|
||||
public final StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
|
||||
String value = si.getAttribute(MODE_KEY);
|
||||
if (value == null) {
|
||||
throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);
|
||||
|
|
@ -137,12 +137,7 @@ public final class Lucene50StoredFieldsFormat extends StoredFieldsFormat {
|
|||
|
||||
@Override
|
||||
public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException {
|
||||
String previous = si.putAttribute(MODE_KEY, mode.name());
|
||||
if (previous != null && previous.equals(mode.name()) == false) {
|
||||
throw new IllegalStateException("found existing value for " + MODE_KEY + " for segment: " + si.name +
|
||||
"old=" + previous + ", new=" + mode.name());
|
||||
}
|
||||
return impl(mode).fieldsWriter(directory, si, context);
|
||||
throw new UnsupportedOperationException("Old codecs may only be used for reading");
|
||||
}
|
||||
|
||||
StoredFieldsFormat impl(Mode mode) {
|
||||
|
|
@ -97,7 +97,7 @@ public class Lucene84Codec extends Codec {
|
|||
}
|
||||
|
||||
@Override
|
||||
public final StoredFieldsFormat storedFieldsFormat() {
|
||||
public StoredFieldsFormat storedFieldsFormat() {
|
||||
return storedFieldsFormat;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ public class Lucene86Codec extends Codec {
|
|||
}
|
||||
|
||||
@Override
|
||||
public final StoredFieldsFormat storedFieldsFormat() {
|
||||
public StoredFieldsFormat storedFieldsFormat() {
|
||||
return storedFieldsFormat;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
|
||||
</head>
|
||||
<body>
|
||||
Lucene 8.6 file format.
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -15,3 +15,4 @@
|
|||
|
||||
org.apache.lucene.codecs.lucene80.Lucene80Codec
|
||||
org.apache.lucene.codecs.lucene84.Lucene84Codec
|
||||
org.apache.lucene.codecs.lucene86.Lucene86Codec
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.codecs.lucene50;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.codecs.StoredFieldsWriter;
|
||||
import org.apache.lucene.index.SegmentInfo;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
|
||||
/**
|
||||
* RW impersonation of Lucene50StoredFieldsFormat.
|
||||
*/
|
||||
public final class Lucene50RWStoredFieldsFormat extends Lucene50StoredFieldsFormat {
|
||||
|
||||
/** No-argument constructor. */
|
||||
public Lucene50RWStoredFieldsFormat() {
|
||||
super();
|
||||
}
|
||||
|
||||
/** Constructor that takes a mode. */
|
||||
public Lucene50RWStoredFieldsFormat(Lucene50StoredFieldsFormat.Mode mode) {
|
||||
super(mode);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException {
|
||||
String previous = si.putAttribute(MODE_KEY, mode.name());
|
||||
if (previous != null && previous.equals(mode.name()) == false) {
|
||||
throw new IllegalStateException("found existing value for " + MODE_KEY + " for segment: " + si.name +
|
||||
"old=" + previous + ", new=" + mode.name());
|
||||
}
|
||||
return impl(mode).fieldsWriter(directory, si, context);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -18,12 +18,12 @@ package org.apache.lucene.codecs.lucene50;
|
|||
|
||||
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.lucene86.Lucene86RWCodec;
|
||||
import org.apache.lucene.index.BaseStoredFieldsFormatTestCase;
|
||||
import org.apache.lucene.util.TestUtil;
|
||||
|
||||
public class TestLucene50StoredFieldsFormat extends BaseStoredFieldsFormatTestCase {
|
||||
@Override
|
||||
protected Codec getCodec() {
|
||||
return TestUtil.getDefaultCodec();
|
||||
return new Lucene86RWCodec();
|
||||
}
|
||||
}
|
||||
|
|
@ -20,7 +20,7 @@ package org.apache.lucene.codecs.lucene50;
|
|||
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
|
||||
import org.apache.lucene.codecs.lucene86.Lucene86Codec;
|
||||
import org.apache.lucene.codecs.lucene86.Lucene86RWCodec;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.StoredField;
|
||||
import org.apache.lucene.index.BaseStoredFieldsFormatTestCase;
|
||||
|
|
@ -32,7 +32,7 @@ import org.apache.lucene.store.Directory;
|
|||
public class TestLucene50StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {
|
||||
@Override
|
||||
protected Codec getCodec() {
|
||||
return new Lucene86Codec(Mode.BEST_COMPRESSION);
|
||||
return new Lucene86RWCodec(Mode.BEST_COMPRESSION);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -43,7 +43,7 @@ public class TestLucene50StoredFieldsFormatHighCompression extends BaseStoredFie
|
|||
Directory dir = newDirectory();
|
||||
for (int i = 0; i < 10; i++) {
|
||||
IndexWriterConfig iwc = newIndexWriterConfig();
|
||||
iwc.setCodec(new Lucene86Codec(RandomPicks.randomFrom(random(), Mode.values())));
|
||||
iwc.setCodec(new Lucene86RWCodec(RandomPicks.randomFrom(random(), Mode.values())));
|
||||
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
|
||||
Document doc = new Document();
|
||||
doc.add(new StoredField("field1", "value1"));
|
||||
|
|
@ -70,7 +70,7 @@ public class TestLucene50StoredFieldsFormatHighCompression extends BaseStoredFie
|
|||
|
||||
public void testInvalidOptions() {
|
||||
expectThrows(NullPointerException.class, () -> {
|
||||
new Lucene86Codec(null);
|
||||
new Lucene86RWCodec(null);
|
||||
});
|
||||
|
||||
expectThrows(NullPointerException.class, () -> {
|
||||
|
|
@ -18,8 +18,7 @@
|
|||
package org.apache.lucene.codecs.lucene70;
|
||||
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.FilterCodec;
|
||||
import org.apache.lucene.codecs.SegmentInfoFormat;
|
||||
import org.apache.lucene.codecs.lucene84.Lucene84RWCodec;
|
||||
import org.apache.lucene.index.BaseSegmentInfoFormatTestCase;
|
||||
import org.apache.lucene.util.Version;
|
||||
|
||||
|
|
@ -32,11 +31,6 @@ public class TestLucene70SegmentInfoFormat extends BaseSegmentInfoFormatTestCase
|
|||
|
||||
@Override
|
||||
protected Codec getCodec() {
|
||||
return new FilterCodec("Lucene84", Codec.forName("Lucene84")) {
|
||||
@Override
|
||||
public SegmentInfoFormat segmentInfoFormat() {
|
||||
return new Lucene70RWSegmentInfoFormat();
|
||||
}
|
||||
};
|
||||
return new Lucene84RWCodec();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ package org.apache.lucene.codecs.lucene84;
|
|||
|
||||
import org.apache.lucene.codecs.PointsFormat;
|
||||
import org.apache.lucene.codecs.SegmentInfoFormat;
|
||||
import org.apache.lucene.codecs.StoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.lucene50.Lucene50RWStoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.lucene60.Lucene60RWPointsFormat;
|
||||
import org.apache.lucene.codecs.lucene70.Lucene70RWSegmentInfoFormat;
|
||||
|
||||
|
|
@ -36,4 +38,9 @@ public class Lucene84RWCodec extends Lucene84Codec {
|
|||
return new Lucene70RWSegmentInfoFormat();
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoredFieldsFormat storedFieldsFormat() {
|
||||
return new Lucene50RWStoredFieldsFormat();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.codecs.lucene86;
|
||||
|
||||
import org.apache.lucene.codecs.StoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.lucene50.Lucene50RWStoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat;
|
||||
|
||||
/**
|
||||
* RW impersonation of {@link Lucene86Codec}.
|
||||
*/
|
||||
public class Lucene86RWCodec extends Lucene86Codec {
|
||||
|
||||
private final StoredFieldsFormat storedFieldsFormat;
|
||||
|
||||
/** No arguments constructor. */
|
||||
public Lucene86RWCodec() {
|
||||
storedFieldsFormat = new Lucene50RWStoredFieldsFormat();
|
||||
}
|
||||
|
||||
/** Constructor that takes a mode. */
|
||||
public Lucene86RWCodec(Lucene50StoredFieldsFormat.Mode mode) {
|
||||
storedFieldsFormat = new Lucene50RWStoredFieldsFormat(mode);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoredFieldsFormat storedFieldsFormat() {
|
||||
return storedFieldsFormat;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -29,7 +29,7 @@ import org.apache.lucene.benchmark.byTask.PerfRunData;
|
|||
import org.apache.lucene.benchmark.byTask.utils.Config;
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.PostingsFormat;
|
||||
import org.apache.lucene.codecs.lucene86.Lucene86Codec;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87Codec;
|
||||
import org.apache.lucene.index.ConcurrentMergeScheduler;
|
||||
import org.apache.lucene.index.IndexCommit;
|
||||
import org.apache.lucene.index.IndexDeletionPolicy;
|
||||
|
|
@ -138,7 +138,7 @@ public class CreateIndexTask extends PerfTask {
|
|||
if (defaultCodec == null && postingsFormat != null) {
|
||||
try {
|
||||
final PostingsFormat postingsFormatChosen = PostingsFormat.forName(postingsFormat);
|
||||
iwConf.setCodec(new Lucene86Codec() {
|
||||
iwConf.setCodec(new Lucene87Codec() {
|
||||
@Override
|
||||
public PostingsFormat getPostingsFormatForField(String field) {
|
||||
return postingsFormatChosen;
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ public abstract class Codec implements NamedSPILoader.NamedSPI {
|
|||
}
|
||||
|
||||
// TODO: should we use this, or maybe a system property is better?
|
||||
static Codec defaultCodec = LOADER.lookup("Lucene86");
|
||||
static Codec defaultCodec = LOADER.lookup("Lucene87");
|
||||
}
|
||||
|
||||
private final String name;
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ package org.apache.lucene.codecs.lucene50;
|
|||
import org.apache.lucene.codecs.CodecUtil;
|
||||
import org.apache.lucene.codecs.TermVectorsFormat;
|
||||
import org.apache.lucene.codecs.compressing.FieldsIndexWriter;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat;
|
||||
import org.apache.lucene.codecs.compressing.CompressionMode;
|
||||
import org.apache.lucene.store.DataOutput;
|
||||
|
|
@ -29,7 +30,7 @@ import org.apache.lucene.util.packed.PackedInts;
|
|||
/**
|
||||
* Lucene 5.0 {@link TermVectorsFormat term vectors format}.
|
||||
* <p>
|
||||
* Very similarly to {@link Lucene50StoredFieldsFormat}, this format is based
|
||||
* Very similarly to {@link Lucene87StoredFieldsFormat}, this format is based
|
||||
* on compressed chunks of data, with document-level granularity so that a
|
||||
* document can never span across distinct chunks. Moreover, data is made as
|
||||
* compact as possible:<ul>
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@
|
|||
* This contains the set of field names used in the index.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@link org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat Stored Field values}.
|
||||
* Stored Field values.
|
||||
* This contains, for each document, a list of attribute-value pairs, where the attributes
|
||||
* are field names. These are used to store auxiliary information about the document, such as
|
||||
* its title, url, or an identifier to access a database. The set of stored fields are what is
|
||||
|
|
@ -250,12 +250,12 @@
|
|||
* <td>Stores information about the fields</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat Field Index}</td>
|
||||
* <td>Field Index</td>
|
||||
* <td>.fdx</td>
|
||||
* <td>Contains pointers to field data</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat Field Data}</td>
|
||||
* <td>Field Data</td>
|
||||
* <td>.fdt</td>
|
||||
* <td>The stored fields for documents</td>
|
||||
* </tr>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,179 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.lucene.codecs.lucene87;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.CompoundFormat;
|
||||
import org.apache.lucene.codecs.DocValuesFormat;
|
||||
import org.apache.lucene.codecs.FieldInfosFormat;
|
||||
import org.apache.lucene.codecs.FilterCodec;
|
||||
import org.apache.lucene.codecs.LiveDocsFormat;
|
||||
import org.apache.lucene.codecs.NormsFormat;
|
||||
import org.apache.lucene.codecs.PointsFormat;
|
||||
import org.apache.lucene.codecs.PostingsFormat;
|
||||
import org.apache.lucene.codecs.SegmentInfoFormat;
|
||||
import org.apache.lucene.codecs.StoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.TermVectorsFormat;
|
||||
import org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat;
|
||||
import org.apache.lucene.codecs.lucene50.Lucene50LiveDocsFormat;
|
||||
import org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat;
|
||||
import org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat;
|
||||
import org.apache.lucene.codecs.lucene80.Lucene80NormsFormat;
|
||||
import org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat;
|
||||
import org.apache.lucene.codecs.lucene86.Lucene86PointsFormat;
|
||||
import org.apache.lucene.codecs.lucene86.Lucene86SegmentInfoFormat;
|
||||
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
|
||||
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
|
||||
|
||||
/**
|
||||
* Implements the Lucene 8.6 index format, with configurable per-field postings
|
||||
* and docvalues formats.
|
||||
* <p>
|
||||
* If you want to reuse functionality of this codec in another codec, extend
|
||||
* {@link FilterCodec}.
|
||||
*
|
||||
* @see org.apache.lucene.codecs.lucene86 package documentation for file format details.
|
||||
*
|
||||
* @lucene.experimental
|
||||
*/
|
||||
public class Lucene87Codec extends Codec {
|
||||
private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
|
||||
private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat();
|
||||
private final SegmentInfoFormat segmentInfosFormat = new Lucene86SegmentInfoFormat();
|
||||
private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
|
||||
private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
|
||||
private final PointsFormat pointsFormat = new Lucene86PointsFormat();
|
||||
private final PostingsFormat defaultFormat;
|
||||
|
||||
private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
|
||||
@Override
|
||||
public PostingsFormat getPostingsFormatForField(String field) {
|
||||
return Lucene87Codec.this.getPostingsFormatForField(field);
|
||||
}
|
||||
};
|
||||
|
||||
private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
|
||||
@Override
|
||||
public DocValuesFormat getDocValuesFormatForField(String field) {
|
||||
return Lucene87Codec.this.getDocValuesFormatForField(field);
|
||||
}
|
||||
};
|
||||
|
||||
private final StoredFieldsFormat storedFieldsFormat;
|
||||
|
||||
/**
|
||||
* Instantiates a new codec.
|
||||
*/
|
||||
public Lucene87Codec() {
|
||||
this(Lucene87StoredFieldsFormat.Mode.BEST_SPEED);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a new codec, specifying the stored fields compression
|
||||
* mode to use.
|
||||
* @param mode stored fields compression mode to use for newly
|
||||
* flushed/merged segments.
|
||||
*/
|
||||
public Lucene87Codec(Lucene87StoredFieldsFormat.Mode mode) {
|
||||
super("Lucene87");
|
||||
this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Objects.requireNonNull(mode));
|
||||
this.defaultFormat = new Lucene84PostingsFormat();
|
||||
}
|
||||
|
||||
@Override
|
||||
public final StoredFieldsFormat storedFieldsFormat() {
|
||||
return storedFieldsFormat;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final TermVectorsFormat termVectorsFormat() {
|
||||
return vectorsFormat;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final PostingsFormat postingsFormat() {
|
||||
return postingsFormat;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final FieldInfosFormat fieldInfosFormat() {
|
||||
return fieldInfosFormat;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final SegmentInfoFormat segmentInfoFormat() {
|
||||
return segmentInfosFormat;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final LiveDocsFormat liveDocsFormat() {
|
||||
return liveDocsFormat;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final CompoundFormat compoundFormat() {
|
||||
return compoundFormat;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final PointsFormat pointsFormat() {
|
||||
return pointsFormat;
|
||||
}
|
||||
|
||||
/** Returns the postings format that should be used for writing
|
||||
* new segments of <code>field</code>.
|
||||
*
|
||||
* The default implementation always returns "Lucene84".
|
||||
* <p>
|
||||
* <b>WARNING:</b> if you subclass, you are responsible for index
|
||||
* backwards compatibility: future version of Lucene are only
|
||||
* guaranteed to be able to read the default implementation.
|
||||
*/
|
||||
public PostingsFormat getPostingsFormatForField(String field) {
|
||||
return defaultFormat;
|
||||
}
|
||||
|
||||
/** Returns the docvalues format that should be used for writing
|
||||
* new segments of <code>field</code>.
|
||||
*
|
||||
* The default implementation always returns "Lucene80".
|
||||
* <p>
|
||||
* <b>WARNING:</b> if you subclass, you are responsible for index
|
||||
* backwards compatibility: future version of Lucene are only
|
||||
* guaranteed to be able to read the default implementation.
|
||||
*/
|
||||
public DocValuesFormat getDocValuesFormatForField(String field) {
|
||||
return defaultDVFormat;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final DocValuesFormat docValuesFormat() {
|
||||
return docValuesFormat;
|
||||
}
|
||||
|
||||
private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80");
|
||||
|
||||
private final NormsFormat normsFormat = new Lucene80NormsFormat();
|
||||
|
||||
@Override
|
||||
public final NormsFormat normsFormat() {
|
||||
return normsFormat;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,378 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.codecs.lucene87;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Objects;
|
||||
import java.util.zip.DataFormatException;
|
||||
import java.util.zip.Deflater;
|
||||
import java.util.zip.Inflater;
|
||||
|
||||
import org.apache.lucene.codecs.StoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.StoredFieldsReader;
|
||||
import org.apache.lucene.codecs.StoredFieldsWriter;
|
||||
import org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.compressing.CompressionMode;
|
||||
import org.apache.lucene.codecs.compressing.Compressor;
|
||||
import org.apache.lucene.codecs.compressing.Decompressor;
|
||||
import org.apache.lucene.index.CorruptIndexException;
|
||||
import org.apache.lucene.index.FieldInfos;
|
||||
import org.apache.lucene.index.SegmentInfo;
|
||||
import org.apache.lucene.index.StoredFieldVisitor;
|
||||
import org.apache.lucene.store.DataInput;
|
||||
import org.apache.lucene.store.DataOutput;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.IOContext;
|
||||
import org.apache.lucene.util.ArrayUtil;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.packed.DirectMonotonicWriter;
|
||||
|
||||
/**
|
||||
* Lucene 8.7 stored fields format.
|
||||
*
|
||||
* <p><b>Principle</b>
|
||||
* <p>This {@link StoredFieldsFormat} compresses blocks of documents in
|
||||
* order to improve the compression ratio compared to document-level
|
||||
* compression. It uses the <a href="http://code.google.com/p/lz4/">LZ4</a>
|
||||
* compression algorithm by default in 16KB blocks, which is fast to compress
|
||||
* and very fast to decompress data. Although the default compression method
|
||||
* that is used ({@link Mode#BEST_SPEED BEST_SPEED}) focuses more on speed than on
|
||||
* compression ratio, it should provide interesting compression ratios
|
||||
* for redundant inputs (such as log files, HTML or plain text). For higher
|
||||
* compression, you can choose ({@link Mode#BEST_COMPRESSION BEST_COMPRESSION}),
|
||||
* which uses the <a href="http://en.wikipedia.org/wiki/DEFLATE">DEFLATE</a>
|
||||
* algorithm with 48kB blocks and shared dictionaries for a better ratio at the
|
||||
* expense of slower performance. These two options can be configured like this:
|
||||
* <pre class="prettyprint">
|
||||
* // the default: for high performance
|
||||
* indexWriterConfig.setCodec(new Lucene87Codec(Mode.BEST_SPEED));
|
||||
* // instead for higher performance (but slower):
|
||||
* // indexWriterConfig.setCodec(new Lucene87Codec(Mode.BEST_COMPRESSION));
|
||||
* </pre>
|
||||
* <p><b>File formats</b>
|
||||
* <p>Stored fields are represented by three files:
|
||||
* <ol>
|
||||
* <li><a id="field_data"></a>
|
||||
* <p>A fields data file (extension <code>.fdt</code>). This file stores a compact
|
||||
* representation of documents in compressed blocks of 16KB or more. When
|
||||
* writing a segment, documents are appended to an in-memory <code>byte[]</code>
|
||||
* buffer. When its size reaches 16KB or more, some metadata about the documents
|
||||
* is flushed to disk, immediately followed by a compressed representation of
|
||||
* the buffer using the
|
||||
* <a href="https://github.com/lz4/lz4">LZ4</a>
|
||||
* <a href="http://fastcompression.blogspot.fr/2011/05/lz4-explained.html">compression format</a>.</p>
|
||||
* <p>Notes
|
||||
* <ul>
|
||||
* <li>When at least one document in a chunk is large enough so that the chunk
|
||||
* is larger than 32KB, the chunk will actually be compressed in several LZ4
|
||||
* blocks of 16KB. This allows {@link StoredFieldVisitor}s which are only
|
||||
* interested in the first fields of a document to not have to decompress 10MB
|
||||
* of data if the document is 10MB, but only 16KB.</li>
|
||||
* <li>Given that the original lengths are written in the metadata of the chunk,
|
||||
* the decompressor can leverage this information to stop decoding as soon as
|
||||
* enough data has been decompressed.</li>
|
||||
* <li>In case documents are incompressible, the overhead of the compression format
|
||||
* is less than 0.5%.</li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* <li><a id="field_index"></a>
|
||||
* <p>A fields index file (extension <code>.fdx</code>). This file stores two
|
||||
* {@link DirectMonotonicWriter monotonic arrays}, one for the first doc IDs of
|
||||
* each block of compressed documents, and another one for the corresponding
|
||||
* offsets on disk. At search time, the array containing doc IDs is
|
||||
* binary-searched in order to find the block that contains the expected doc ID,
|
||||
* and the associated offset on disk is retrieved from the second array.</p>
|
||||
* <li><a id="field_meta"></a>
|
||||
* <p>A fields meta file (extension <code>.fdm</code>). This file stores metadata
|
||||
* about the monotonic arrays stored in the index file.</p>
|
||||
* </li>
|
||||
* </ol>
|
||||
* <p><b>Known limitations</b>
|
||||
* <p>This {@link StoredFieldsFormat} does not support individual documents
|
||||
* larger than (<code>2<sup>31</sup> - 2<sup>14</sup></code>) bytes.
|
||||
* @lucene.experimental
|
||||
*/
|
||||
public class Lucene87StoredFieldsFormat extends StoredFieldsFormat {
|
||||
|
||||
/** Configuration option for stored fields. */
|
||||
public static enum Mode {
|
||||
/** Trade compression ratio for retrieval speed. */
|
||||
BEST_SPEED,
|
||||
/** Trade retrieval speed for compression ratio. */
|
||||
BEST_COMPRESSION
|
||||
}
|
||||
|
||||
/** Attribute key for compression mode. */
|
||||
public static final String MODE_KEY = Lucene87StoredFieldsFormat.class.getSimpleName() + ".mode";
|
||||
|
||||
final Mode mode;
|
||||
|
||||
/** Stored fields format with default options */
|
||||
public Lucene87StoredFieldsFormat() {
|
||||
this(Mode.BEST_SPEED);
|
||||
}
|
||||
|
||||
/** Stored fields format with specified mode */
|
||||
public Lucene87StoredFieldsFormat(Mode mode) {
|
||||
this.mode = Objects.requireNonNull(mode);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
|
||||
String value = si.getAttribute(MODE_KEY);
|
||||
if (value == null) {
|
||||
throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);
|
||||
}
|
||||
Mode mode = Mode.valueOf(value);
|
||||
return impl(mode).fieldsReader(directory, si, fn, context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException {
|
||||
String previous = si.putAttribute(MODE_KEY, mode.name());
|
||||
if (previous != null && previous.equals(mode.name()) == false) {
|
||||
throw new IllegalStateException("found existing value for " + MODE_KEY + " for segment: " + si.name +
|
||||
"old=" + previous + ", new=" + mode.name());
|
||||
}
|
||||
return impl(mode).fieldsWriter(directory, si, context);
|
||||
}
|
||||
|
||||
StoredFieldsFormat impl(Mode mode) {
|
||||
switch (mode) {
|
||||
case BEST_SPEED:
|
||||
return new CompressingStoredFieldsFormat("Lucene87StoredFieldsFastData", CompressionMode.FAST, 16*1024, 128, 10);
|
||||
case BEST_COMPRESSION:
|
||||
return new CompressingStoredFieldsFormat("Lucene87StoredFieldsHighData", BEST_COMPRESSION_MODE, BEST_COMPRESSION_BLOCK_LENGTH, 512, 10);
|
||||
default: throw new AssertionError();
|
||||
}
|
||||
}
|
||||
|
||||
// 8kB seems to be a good trade-off between higher compression rates by not
|
||||
// having to fully bootstrap a dictionary, and indexing rate by not spending
|
||||
// too much CPU initializing data-structures to find strings in this preset
|
||||
// dictionary.
|
||||
private static final int BEST_COMPRESSION_DICT_LENGTH = 8 * 1024;
|
||||
// 48kB seems like a nice trade-off because it's small enough to keep
|
||||
// retrieval fast, yet sub blocks can find strings in a window of 26kB of
|
||||
// data on average (the window grows from 8kB to 32kB in the first 24kB, and
|
||||
// then DEFLATE can use 32kB for the last 24kB) which is close enough to the
|
||||
// maximum window length of DEFLATE of 32kB.
|
||||
private static final int BEST_COMPRESSION_SUB_BLOCK_LENGTH = 48 * 1024;
|
||||
// We shoot for 10 sub blocks per block, which should hopefully amortize the
|
||||
// space overhead of having the first 8kB compressed without any preset dict,
|
||||
// and then remove 8kB in order to avoid creating a tiny 11th sub block if
|
||||
// documents are small.
|
||||
private static final int BEST_COMPRESSION_BLOCK_LENGTH = BEST_COMPRESSION_DICT_LENGTH + 10 * BEST_COMPRESSION_SUB_BLOCK_LENGTH - 8 * 1024;
|
||||
|
||||
/** Compression mode for {@link Mode#BEST_COMPRESSION} */
|
||||
public static final DeflateWithPresetDict BEST_COMPRESSION_MODE = new DeflateWithPresetDict(BEST_COMPRESSION_DICT_LENGTH, BEST_COMPRESSION_SUB_BLOCK_LENGTH);
|
||||
|
||||
/**
|
||||
* A compression mode that trades speed for compression ratio. Although
|
||||
* compression and decompression might be slow, this compression mode should
|
||||
* provide a good compression ratio. This mode might be interesting if/when
|
||||
* your index size is much bigger than your OS cache.
|
||||
*/
|
||||
public static class DeflateWithPresetDict extends CompressionMode {
|
||||
|
||||
private final int dictLength, subBlockLength;
|
||||
|
||||
/** Sole constructor. */
|
||||
public DeflateWithPresetDict(int dictLength, int subBlockLength) {
|
||||
this.dictLength = dictLength;
|
||||
this.subBlockLength = subBlockLength;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Compressor newCompressor() {
|
||||
// notes:
|
||||
// 3 is the highest level that doesn't have lazy match evaluation
|
||||
// 6 is the default, higher than that is just a waste of cpu
|
||||
return new DeflateWithPresetDictCompressor(6, dictLength, subBlockLength);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Decompressor newDecompressor() {
|
||||
return new DeflateWithPresetDictDecompressor();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "BEST_COMPRESSION";
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
private static final class DeflateWithPresetDictDecompressor extends Decompressor {
|
||||
|
||||
byte[] compressed;
|
||||
|
||||
DeflateWithPresetDictDecompressor() {
|
||||
compressed = new byte[0];
|
||||
}
|
||||
|
||||
private void doDecompress(DataInput in, Inflater decompressor, BytesRef bytes) throws IOException {
|
||||
final int compressedLength = in.readVInt();
|
||||
if (compressedLength == 0) {
|
||||
return;
|
||||
}
|
||||
// pad with extra "dummy byte": see javadocs for using Inflater(true)
|
||||
// we do it for compliance, but it's unnecessary for years in zlib.
|
||||
final int paddedLength = compressedLength + 1;
|
||||
compressed = ArrayUtil.grow(compressed, paddedLength);
|
||||
in.readBytes(compressed, 0, compressedLength);
|
||||
compressed[compressedLength] = 0; // explicitly set dummy byte to 0
|
||||
|
||||
// extra "dummy byte"
|
||||
decompressor.setInput(compressed, 0, paddedLength);
|
||||
try {
|
||||
bytes.length += decompressor.inflate(bytes.bytes, bytes.length, bytes.bytes.length - bytes.length);
|
||||
} catch (DataFormatException e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
if (decompressor.finished() == false) {
|
||||
throw new CorruptIndexException("Invalid decoder state: needsInput=" + decompressor.needsInput()
|
||||
+ ", needsDict=" + decompressor.needsDictionary(), in);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException {
|
||||
assert offset + length <= originalLength;
|
||||
if (length == 0) {
|
||||
bytes.length = 0;
|
||||
return;
|
||||
}
|
||||
final int dictLength = in.readVInt();
|
||||
final int blockLength = in.readVInt();
|
||||
bytes.bytes = ArrayUtil.grow(bytes.bytes, dictLength);
|
||||
bytes.offset = bytes.length = 0;
|
||||
|
||||
final Inflater decompressor = new Inflater(true);
|
||||
try {
|
||||
// Read the dictionary
|
||||
doDecompress(in, decompressor, bytes);
|
||||
if (dictLength != bytes.length) {
|
||||
throw new CorruptIndexException("Unexpected dict length", in);
|
||||
}
|
||||
|
||||
int offsetInBlock = dictLength;
|
||||
int offsetInBytesRef = offset;
|
||||
|
||||
// Skip unneeded blocks
|
||||
while (offsetInBlock + blockLength < offset) {
|
||||
final int compressedLength = in.readVInt();
|
||||
in.skipBytes(compressedLength);
|
||||
offsetInBlock += blockLength;
|
||||
offsetInBytesRef -= blockLength;
|
||||
}
|
||||
|
||||
// Read blocks that intersect with the interval we need
|
||||
while (offsetInBlock < offset + length) {
|
||||
bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + blockLength);
|
||||
decompressor.reset();
|
||||
decompressor.setDictionary(bytes.bytes, 0, dictLength);
|
||||
doDecompress(in, decompressor, bytes);
|
||||
offsetInBlock += blockLength;
|
||||
}
|
||||
|
||||
bytes.offset = offsetInBytesRef;
|
||||
bytes.length = length;
|
||||
assert bytes.isValid();
|
||||
} finally {
|
||||
decompressor.end();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Decompressor clone() {
|
||||
return new DeflateWithPresetDictDecompressor();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static class DeflateWithPresetDictCompressor extends Compressor {
|
||||
|
||||
final int dictLength;
|
||||
final int blockLength;
|
||||
final Deflater compressor;
|
||||
byte[] compressed;
|
||||
boolean closed;
|
||||
|
||||
DeflateWithPresetDictCompressor(int level, int dictLength, int blockLength) {
|
||||
compressor = new Deflater(level, true);
|
||||
compressed = new byte[64];
|
||||
this.dictLength = dictLength;
|
||||
this.blockLength = blockLength;
|
||||
}
|
||||
|
||||
private void doCompress(byte[] bytes, int off, int len, DataOutput out) throws IOException {
|
||||
if (len == 0) {
|
||||
out.writeVInt(0);
|
||||
return;
|
||||
}
|
||||
compressor.setInput(bytes, off, len);
|
||||
compressor.finish();
|
||||
if (compressor.needsInput()) {
|
||||
throw new IllegalStateException();
|
||||
}
|
||||
|
||||
int totalCount = 0;
|
||||
for (;;) {
|
||||
final int count = compressor.deflate(compressed, totalCount, compressed.length - totalCount);
|
||||
totalCount += count;
|
||||
assert totalCount <= compressed.length;
|
||||
if (compressor.finished()) {
|
||||
break;
|
||||
} else {
|
||||
compressed = ArrayUtil.grow(compressed);
|
||||
}
|
||||
}
|
||||
|
||||
out.writeVInt(totalCount);
|
||||
out.writeBytes(compressed, totalCount);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void compress(byte[] bytes, int off, int len, DataOutput out) throws IOException {
|
||||
final int dictLength = Math.min(this.dictLength, len);
|
||||
out.writeVInt(dictLength);
|
||||
out.writeVInt(blockLength);
|
||||
final int end = off + len;
|
||||
|
||||
// Compress the dictionary first
|
||||
compressor.reset();
|
||||
doCompress(bytes, off, dictLength, out);
|
||||
|
||||
// And then sub blocks
|
||||
for (int start = off + dictLength; start < end; start += blockLength) {
|
||||
compressor.reset();
|
||||
compressor.setDictionary(bytes, off, dictLength);
|
||||
doCompress(bytes, start, Math.min(blockLength, off + len - start), out);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (closed == false) {
|
||||
compressor.end();
|
||||
closed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,416 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Lucene 8.7 file format.
|
||||
*
|
||||
* <h2>Apache Lucene - Index File Formats</h2>
|
||||
* <div>
|
||||
* <ul>
|
||||
* <li><a href="#Introduction">Introduction</a></li>
|
||||
* <li><a href="#Definitions">Definitions</a>
|
||||
* <ul>
|
||||
* <li><a href="#Inverted_Indexing">Inverted Indexing</a></li>
|
||||
* <li><a href="#Types_of_Fields">Types of Fields</a></li>
|
||||
* <li><a href="#Segments">Segments</a></li>
|
||||
* <li><a href="#Document_Numbers">Document Numbers</a></li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* <li><a href="#Overview">Index Structure Overview</a></li>
|
||||
* <li><a href="#File_Naming">File Naming</a></li>
|
||||
* <li><a href="#file-names">Summary of File Extensions</a>
|
||||
* <ul>
|
||||
* <li><a href="#Lock_File">Lock File</a></li>
|
||||
* <li><a href="#History">History</a></li>
|
||||
* <li><a href="#Limitations">Limitations</a></li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* </ul>
|
||||
* </div>
|
||||
* <a id="Introduction"></a>
|
||||
* <h3>Introduction</h3>
|
||||
* <div>
|
||||
* <p>This document defines the index file formats used in this version of Lucene.
|
||||
* If you are using a different version of Lucene, please consult the copy of
|
||||
* <code>docs/</code> that was distributed with
|
||||
* the version you are using.</p>
|
||||
* <p>This document attempts to provide a high-level definition of the Apache
|
||||
* Lucene file formats.</p>
|
||||
* </div>
|
||||
* <a id="Definitions"></a>
|
||||
* <h3>Definitions</h3>
|
||||
* <div>
|
||||
* <p>The fundamental concepts in Lucene are index, document, field and term.</p>
|
||||
* <p>An index contains a sequence of documents.</p>
|
||||
* <ul>
|
||||
* <li>A document is a sequence of fields.</li>
|
||||
* <li>A field is a named sequence of terms.</li>
|
||||
* <li>A term is a sequence of bytes.</li>
|
||||
* </ul>
|
||||
* <p>The same sequence of bytes in two different fields is considered a different
|
||||
* term. Thus terms are represented as a pair: the string naming the field, and the
|
||||
* bytes within the field.</p>
|
||||
* <a id="Inverted_Indexing"></a>
|
||||
* <h4>Inverted Indexing</h4>
|
||||
* <p>The index stores statistics about terms in order to make term-based search
|
||||
* more efficient. Lucene's index falls into the family of indexes known as an
|
||||
* <i>inverted index.</i> This is because it can list, for a term, the documents
|
||||
* that contain it. This is the inverse of the natural relationship, in which
|
||||
* documents list terms.</p>
|
||||
* <a id="Types_of_Fields"></a>
|
||||
* <h4>Types of Fields</h4>
|
||||
* <p>In Lucene, fields may be <i>stored</i>, in which case their text is stored
|
||||
* in the index literally, in a non-inverted manner. Fields that are inverted are
|
||||
* called <i>indexed</i>. A field may be both stored and indexed.</p>
|
||||
* <p>The text of a field may be <i>tokenized</i> into terms to be indexed, or the
|
||||
* text of a field may be used literally as a term to be indexed. Most fields are
|
||||
* tokenized, but sometimes it is useful for certain identifier fields to be
|
||||
* indexed literally.</p>
|
||||
* <p>See the {@link org.apache.lucene.document.Field Field}
|
||||
* java docs for more information on Fields.</p>
|
||||
* <a id="Segments"></a>
|
||||
* <h4>Segments</h4>
|
||||
* <p>Lucene indexes may be composed of multiple sub-indexes, or <i>segments</i>.
|
||||
* Each segment is a fully independent index, which could be searched separately.
|
||||
* Indexes evolve by:</p>
|
||||
* <ol>
|
||||
* <li>Creating new segments for newly added documents.</li>
|
||||
* <li>Merging existing segments.</li>
|
||||
* </ol>
|
||||
* <p>Searches may involve multiple segments and/or multiple indexes, each index
|
||||
* potentially composed of a set of segments.</p>
|
||||
* <a id="Document_Numbers"></a>
|
||||
* <h4>Document Numbers</h4>
|
||||
* <p>Internally, Lucene refers to documents by an integer <i>document number</i>.
|
||||
* The first document added to an index is numbered zero, and each subsequent
|
||||
* document added gets a number one greater than the previous.</p>
|
||||
* <p>Note that a document's number may change, so caution should be taken when
|
||||
* storing these numbers outside of Lucene. In particular, numbers may change in
|
||||
* the following situations:</p>
|
||||
* <ul>
|
||||
* <li>
|
||||
* <p>The numbers stored in each segment are unique only within the segment, and
|
||||
* must be converted before they can be used in a larger context. The standard
|
||||
* technique is to allocate each segment a range of values, based on the range of
|
||||
* numbers used in that segment. To convert a document number from a segment to an
|
||||
* external value, the segment's <i>base</i> document number is added. To convert
|
||||
* an external value back to a segment-specific value, the segment is identified
|
||||
* by the range that the external value is in, and the segment's base value is
|
||||
* subtracted. For example two five document segments might be combined, so that
|
||||
* the first segment has a base value of zero, and the second of five. Document
|
||||
* three from the second segment would have an external value of eight.</p>
|
||||
* </li>
|
||||
* <li>
|
||||
* <p>When documents are deleted, gaps are created in the numbering. These are
|
||||
* eventually removed as the index evolves through merging. Deleted documents are
|
||||
* dropped when segments are merged. A freshly-merged segment thus has no gaps in
|
||||
* its numbering.</p>
|
||||
* </li>
|
||||
* </ul>
|
||||
* </div>
|
||||
* <a id="Overview"></a>
|
||||
* <h3>Index Structure Overview</h3>
|
||||
* <div>
|
||||
* <p>Each segment index maintains the following:</p>
|
||||
* <ul>
|
||||
* <li>
|
||||
* {@link org.apache.lucene.codecs.lucene86.Lucene86SegmentInfoFormat Segment info}.
|
||||
* This contains metadata about a segment, such as the number of documents,
|
||||
* what files it uses, and information about how the segment is sorted
|
||||
* </li>
|
||||
* <li>
|
||||
* {@link org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat Field names}.
|
||||
* This contains the set of field names used in the index.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@link org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat Stored Field values}.
|
||||
* This contains, for each document, a list of attribute-value pairs, where the attributes
|
||||
* are field names. These are used to store auxiliary information about the document, such as
|
||||
* its title, url, or an identifier to access a database. The set of stored fields are what is
|
||||
* returned for each hit when searching. This is keyed by document number.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Term dictionary}.
|
||||
* A dictionary containing all of the terms used in all of the
|
||||
* indexed fields of all of the documents. The dictionary also contains the number
|
||||
* of documents which contain the term, and pointers to the term's frequency and
|
||||
* proximity data.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Term Frequency data}.
|
||||
* For each term in the dictionary, the numbers of all the
|
||||
* documents that contain that term, and the frequency of the term in that
|
||||
* document, unless frequencies are omitted ({@link org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
|
||||
* </li>
|
||||
* <li>
|
||||
* {@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Term Proximity data}.
|
||||
* For each term in the dictionary, the positions that the
|
||||
* term occurs in each document. Note that this will not exist if all fields in
|
||||
* all documents omit position data.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@link org.apache.lucene.codecs.lucene80.Lucene80NormsFormat Normalization factors}.
|
||||
* For each field in each document, a value is stored
|
||||
* that is multiplied into the score for hits on that field.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@link org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat Term Vectors}.
|
||||
* For each field in each document, the term vector (sometimes
|
||||
* called document vector) may be stored. A term vector consists of term text and
|
||||
* term frequency. To add Term Vectors to your index see the
|
||||
* {@link org.apache.lucene.document.Field Field} constructors
|
||||
* </li>
|
||||
* <li>
|
||||
* {@link org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat Per-document values}.
|
||||
* Like stored values, these are also keyed by document
|
||||
* number, but are generally intended to be loaded into main memory for fast
|
||||
* access. Whereas stored values are generally intended for summary results from
|
||||
* searches, per-document values are useful for things like scoring factors.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@link org.apache.lucene.codecs.lucene50.Lucene50LiveDocsFormat Live documents}.
|
||||
* An optional file indicating which documents are live.
|
||||
* </li>
|
||||
* <li>
|
||||
* {@link org.apache.lucene.codecs.lucene86.Lucene86PointsFormat Point values}.
|
||||
* Optional pair of files, recording dimensionally indexed fields, to enable fast
|
||||
* numeric range filtering and large numeric values like BigInteger and BigDecimal (1D)
|
||||
* and geographic shape intersection (2D, 3D).
|
||||
* </li>
|
||||
* </ul>
|
||||
* <p>Details on each of these are provided in their linked pages.</p>
|
||||
* </div>
|
||||
* <a id="File_Naming"></a>
|
||||
* <h3>File Naming</h3>
|
||||
* <div>
|
||||
* <p>All files belonging to a segment have the same name with varying extensions.
|
||||
* The extensions correspond to the different file formats described below. When
|
||||
* using the Compound File format (default for small segments) these files (except
|
||||
* for the Segment info file, the Lock file, and Deleted documents file) are collapsed
|
||||
* into a single .cfs file (see below for details)</p>
|
||||
* <p>Typically, all segments in an index are stored in a single directory,
|
||||
* although this is not required.</p>
|
||||
* <p>File names are never re-used. That is, when any file is saved
|
||||
* to the Directory it is given a never before used filename. This is achieved
|
||||
* using a simple generations approach. For example, the first segments file is
|
||||
* segments_1, then segments_2, etc. The generation is a sequential long integer
|
||||
* represented in alpha-numeric (base 36) form.</p>
|
||||
* </div>
|
||||
* <a id="file-names"></a>
|
||||
* <h3>Summary of File Extensions</h3>
|
||||
* <div>
|
||||
* <p>The following table summarizes the names and extensions of the files in
|
||||
* Lucene:</p>
|
||||
* <table class="padding4" style="border-spacing: 1px; border-collapse: separate">
|
||||
* <caption>lucene filenames by extension</caption>
|
||||
* <tr>
|
||||
* <th>Name</th>
|
||||
* <th>Extension</th>
|
||||
* <th>Brief Description</th>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.index.SegmentInfos Segments File}</td>
|
||||
* <td>segments_N</td>
|
||||
* <td>Stores information about a commit point</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td><a href="#Lock_File">Lock File</a></td>
|
||||
* <td>write.lock</td>
|
||||
* <td>The Write lock prevents multiple IndexWriters from writing to the same
|
||||
* file.</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene86.Lucene86SegmentInfoFormat Segment Info}</td>
|
||||
* <td>.si</td>
|
||||
* <td>Stores metadata about a segment</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat Compound File}</td>
|
||||
* <td>.cfs, .cfe</td>
|
||||
* <td>An optional "virtual" file consisting of all the other index files for
|
||||
* systems that frequently run out of file handles.</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat Fields}</td>
|
||||
* <td>.fnm</td>
|
||||
* <td>Stores information about the fields</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat Field Index}</td>
|
||||
* <td>.fdx</td>
|
||||
* <td>Contains pointers to field data</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat Field Data}</td>
|
||||
* <td>.fdt</td>
|
||||
* <td>The stored fields for documents</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Term Dictionary}</td>
|
||||
* <td>.tim</td>
|
||||
* <td>The term dictionary, stores term info</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Term Index}</td>
|
||||
* <td>.tip</td>
|
||||
* <td>The index into the Term Dictionary</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Frequencies}</td>
|
||||
* <td>.doc</td>
|
||||
* <td>Contains the list of docs which contain each term along with frequency</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Positions}</td>
|
||||
* <td>.pos</td>
|
||||
* <td>Stores position information about where a term occurs in the index</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Payloads}</td>
|
||||
* <td>.pay</td>
|
||||
* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene80.Lucene80NormsFormat Norms}</td>
|
||||
* <td>.nvd, .nvm</td>
|
||||
* <td>Encodes length and boost factors for docs and fields</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat Per-Document Values}</td>
|
||||
* <td>.dvd, .dvm</td>
|
||||
* <td>Encodes additional scoring factors or other per-document information.</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat Term Vector Index}</td>
|
||||
* <td>.tvx</td>
|
||||
* <td>Stores offset into the document data file</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat Term Vector Data}</td>
|
||||
* <td>.tvd</td>
|
||||
* <td>Contains term vector data.</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene50.Lucene50LiveDocsFormat Live Documents}</td>
|
||||
* <td>.liv</td>
|
||||
* <td>Info about what documents are live</td>
|
||||
* </tr>
|
||||
* <tr>
|
||||
* <td>{@link org.apache.lucene.codecs.lucene86.Lucene86PointsFormat Point values}</td>
|
||||
* <td>.dii, .dim</td>
|
||||
* <td>Holds indexed points, if any</td>
|
||||
* </tr>
|
||||
* </table>
|
||||
* </div>
|
||||
* <a id="Lock_File"></a>
|
||||
* <h3>Lock File</h3>
|
||||
* The write lock, which is stored in the index directory by default, is named
|
||||
* "write.lock". If the lock directory is different from the index directory then
|
||||
* the write lock will be named "XXXX-write.lock" where XXXX is a unique prefix
|
||||
* derived from the full path to the index directory. When this file is present, a
|
||||
* writer is currently modifying the index (adding or removing documents). This
|
||||
* lock file ensures that only one writer is modifying the index at a time.
|
||||
* <a id="History"></a>
|
||||
* <h3>History</h3>
|
||||
* <p>Compatibility notes are provided in this document, describing how file
|
||||
* formats have changed from prior versions:</p>
|
||||
* <ul>
|
||||
* <li>In version 2.1, the file format was changed to allow lock-less commits (ie,
|
||||
* no more commit lock). The change is fully backwards compatible: you can open a
|
||||
* pre-2.1 index for searching or adding/deleting of docs. When the new segments
|
||||
* file is saved (committed), it will be written in the new file format (meaning
|
||||
* no specific "upgrade" process is needed). But note that once a commit has
|
||||
* occurred, pre-2.1 Lucene will not be able to read the index.</li>
|
||||
* <li>In version 2.3, the file format was changed to allow segments to share a
|
||||
* single set of doc store (vectors & stored fields) files. This allows for
|
||||
* faster indexing in certain cases. The change is fully backwards compatible (in
|
||||
* the same way as the lock-less commits change in 2.1).</li>
|
||||
* <li>In version 2.4, Strings are now written as true UTF-8 byte sequence, not
|
||||
* Java's modified UTF-8. See <a href="http://issues.apache.org/jira/browse/LUCENE-510">
|
||||
* LUCENE-510</a> for details.</li>
|
||||
* <li>In version 2.9, an optional opaque Map<String,String> CommitUserData
|
||||
* may be passed to IndexWriter's commit methods (and later retrieved), which is
|
||||
* recorded in the segments_N file. See <a href="http://issues.apache.org/jira/browse/LUCENE-1382">
|
||||
* LUCENE-1382</a> for details. Also,
|
||||
* diagnostics were added to each segment written recording details about why it
|
||||
* was written (due to flush, merge; which OS/JRE was used; etc.). See issue
|
||||
* <a href="http://issues.apache.org/jira/browse/LUCENE-1654">LUCENE-1654</a> for details.</li>
|
||||
* <li>In version 3.0, compressed fields are no longer written to the index (they
|
||||
* can still be read, but on merge the new segment will write them, uncompressed).
|
||||
* See issue <a href="http://issues.apache.org/jira/browse/LUCENE-1960">LUCENE-1960</a>
|
||||
* for details.</li>
|
||||
* <li>In version 3.1, segments records the code version that created them. See
|
||||
* <a href="http://issues.apache.org/jira/browse/LUCENE-2720">LUCENE-2720</a> for details.
|
||||
* Additionally segments track explicitly whether or not they have term vectors.
|
||||
* See <a href="http://issues.apache.org/jira/browse/LUCENE-2811">LUCENE-2811</a>
|
||||
* for details.</li>
|
||||
* <li>In version 3.2, numeric fields are written as natively to stored fields
|
||||
* file, previously they were stored in text format only.</li>
|
||||
* <li>In version 3.4, fields can omit position data while still indexing term
|
||||
* frequencies.</li>
|
||||
* <li>In version 4.0, the format of the inverted index became extensible via
|
||||
* the {@link org.apache.lucene.codecs.Codec Codec} api. Fast per-document storage
|
||||
* ({@code DocValues}) was introduced. Normalization factors need no longer be a
|
||||
* single byte, they can be any {@link org.apache.lucene.index.NumericDocValues NumericDocValues}.
|
||||
* Terms need not be unicode strings, they can be any byte sequence. Term offsets
|
||||
* can optionally be indexed into the postings lists. Payloads can be stored in the
|
||||
* term vectors.</li>
|
||||
* <li>In version 4.1, the format of the postings list changed to use either
|
||||
* of FOR compression or variable-byte encoding, depending upon the frequency
|
||||
* of the term. Terms appearing only once were changed to inline directly into
|
||||
* the term dictionary. Stored fields are compressed by default. </li>
|
||||
* <li>In version 4.2, term vectors are compressed by default. DocValues has
|
||||
* a new multi-valued type (SortedSet), that can be used for faceting/grouping/joining
|
||||
* on multi-valued fields.</li>
|
||||
* <li>In version 4.5, DocValues were extended to explicitly represent missing values.</li>
|
||||
* <li>In version 4.6, FieldInfos were extended to support per-field DocValues generation, to
|
||||
* allow updating NumericDocValues fields.</li>
|
||||
* <li>In version 4.8, checksum footers were added to the end of each index file
|
||||
* for improved data integrity. Specifically, the last 8 bytes of every index file
|
||||
* contain the zlib-crc32 checksum of the file.</li>
|
||||
* <li>In version 4.9, DocValues has a new multi-valued numeric type (SortedNumeric)
|
||||
* that is suitable for faceting/sorting/analytics.
|
||||
* <li>In version 5.4, DocValues have been improved to store more information on disk:
|
||||
* addresses for binary fields and ord indexes for multi-valued fields.
|
||||
* <li>In version 6.0, Points were added, for multi-dimensional range/distance search.
|
||||
* <li>In version 6.2, new Segment info format that reads/writes the index sort, to support index sorting.
|
||||
* <li>In version 7.0, DocValues have been improved to better support sparse doc values
|
||||
* thanks to an iterator API.</li>
|
||||
* <li>In version 8.0, postings have been enhanced to record, for each block of
|
||||
* doc ids, the (term freq, normalization factor) pairs that may trigger the
|
||||
* maximum score of the block. This information is recorded alongside skip data
|
||||
* in order to be able to skip blocks of doc ids if they may not produce high
|
||||
* enough scores.
|
||||
* Additionally doc values and norms has been extended with jump-tables to make access O(1)
|
||||
* instead of O(n), where n is the number of elements to skip when advancing in the data.</li>
|
||||
* <li>In version 8.4, postings, positions, offsets and payload lengths have move to a more
|
||||
* performant encoding that is vectorized.</li>
|
||||
* <li>In version 8.6, index sort serialization is delegated to the sorts themselves, to
|
||||
* allow user-defined sorts to be used</li>
|
||||
* </ul>
|
||||
* <a id="Limitations"></a>
|
||||
* <h3>Limitations</h3>
|
||||
* <div>
|
||||
* <p>Lucene uses a Java <code>int</code> to refer to
|
||||
* document numbers, and the index file format uses an <code>Int32</code>
|
||||
* on-disk to store document numbers. This is a limitation
|
||||
* of both the index file format and the current implementation. Eventually these
|
||||
* should be replaced with either <code>UInt64</code> values, or
|
||||
* better yet, {@link org.apache.lucene.store.DataOutput#writeVInt VInt} values which have no limit.</p>
|
||||
* </div>
|
||||
*/
|
||||
package org.apache.lucene.codecs.lucene87;
|
||||
|
|
@ -13,4 +13,4 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
org.apache.lucene.codecs.lucene86.Lucene86Codec
|
||||
org.apache.lucene.codecs.lucene87.Lucene87Codec
|
||||
|
|
|
|||
|
|
@ -18,14 +18,14 @@ package org.apache.lucene.codecs.lucene80;
|
|||
|
||||
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.lucene86.Lucene86Codec;
|
||||
import org.apache.lucene.index.BaseNormsFormatTestCase;
|
||||
import org.apache.lucene.util.TestUtil;
|
||||
|
||||
/**
|
||||
* Tests Lucene80NormsFormat
|
||||
*/
|
||||
public class TestLucene80NormsFormat extends BaseNormsFormatTestCase {
|
||||
private final Codec codec = new Lucene86Codec();
|
||||
private final Codec codec = TestUtil.getDefaultCodec();
|
||||
|
||||
@Override
|
||||
protected Codec getCodec() {
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ public class TestLucene86PointsFormat extends BasePointsFormatTestCase {
|
|||
|
||||
public TestLucene86PointsFormat() {
|
||||
// standard issue
|
||||
Codec defaultCodec = new Lucene86Codec();
|
||||
Codec defaultCodec = TestUtil.getDefaultCodec();
|
||||
if (random().nextBoolean()) {
|
||||
// randomize parameters
|
||||
maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 500);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.codecs.lucene87;
|
||||
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.index.BaseStoredFieldsFormatTestCase;
|
||||
import org.apache.lucene.util.TestUtil;
|
||||
|
||||
public class TestLucene87StoredFieldsFormat extends BaseStoredFieldsFormatTestCase {
|
||||
@Override
|
||||
protected Codec getCodec() {
|
||||
return TestUtil.getDefaultCodec();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.codecs.lucene87;
|
||||
|
||||
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.Mode;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.StoredField;
|
||||
import org.apache.lucene.index.BaseStoredFieldsFormatTestCase;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.IndexWriterConfig;
|
||||
import org.apache.lucene.store.Directory;
|
||||
|
||||
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
|
||||
|
||||
public class TestLucene87StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {
|
||||
@Override
|
||||
protected Codec getCodec() {
|
||||
return new Lucene87Codec(Mode.BEST_COMPRESSION);
|
||||
}
|
||||
|
||||
/**
|
||||
* Change compression params (leaving it the same for old segments)
|
||||
* and tests that nothing breaks.
|
||||
*/
|
||||
public void testMixedCompressions() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
for (int i = 0; i < 10; i++) {
|
||||
IndexWriterConfig iwc = newIndexWriterConfig();
|
||||
iwc.setCodec(new Lucene87Codec(RandomPicks.randomFrom(random(), Mode.values())));
|
||||
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
|
||||
Document doc = new Document();
|
||||
doc.add(new StoredField("field1", "value1"));
|
||||
doc.add(new StoredField("field2", "value2"));
|
||||
iw.addDocument(doc);
|
||||
if (random().nextInt(4) == 0) {
|
||||
iw.forceMerge(1);
|
||||
}
|
||||
iw.commit();
|
||||
iw.close();
|
||||
}
|
||||
|
||||
DirectoryReader ir = DirectoryReader.open(dir);
|
||||
assertEquals(10, ir.numDocs());
|
||||
for (int i = 0; i < 10; i++) {
|
||||
Document doc = ir.document(i);
|
||||
assertEquals("value1", doc.get("field1"));
|
||||
assertEquals("value2", doc.get("field2"));
|
||||
}
|
||||
ir.close();
|
||||
// checkindex
|
||||
dir.close();
|
||||
}
|
||||
|
||||
public void testInvalidOptions() {
|
||||
expectThrows(NullPointerException.class, () -> {
|
||||
new Lucene87Codec(null);
|
||||
});
|
||||
|
||||
expectThrows(NullPointerException.class, () -> {
|
||||
new Lucene87StoredFieldsFormat(null);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.codecs.lucene87;
|
||||
|
||||
/**
|
||||
* Test the merge instance of the Lucene50 stored fields format.
|
||||
*/
|
||||
public class TestLucene87StoredFieldsFormatMergeInstance extends TestLucene87StoredFieldsFormat {
|
||||
|
||||
@Override
|
||||
protected boolean shouldTestMergeInstance() {
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -396,7 +396,7 @@ public class TestPointValues extends LuceneTestCase {
|
|||
public void testDifferentCodecs1() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
|
||||
iwc.setCodec(Codec.forName("Lucene86"));
|
||||
iwc.setCodec(TestUtil.getDefaultCodec());
|
||||
IndexWriter w = new IndexWriter(dir, iwc);
|
||||
Document doc = new Document();
|
||||
doc.add(new IntPoint("int", 1));
|
||||
|
|
@ -427,7 +427,7 @@ public class TestPointValues extends LuceneTestCase {
|
|||
w.close();
|
||||
|
||||
iwc = new IndexWriterConfig(new MockAnalyzer(random()));
|
||||
iwc.setCodec(Codec.forName("Lucene86"));
|
||||
iwc.setCodec(TestUtil.getDefaultCodec());
|
||||
w = new IndexWriter(dir, iwc);
|
||||
doc = new Document();
|
||||
doc.add(new IntPoint("int", 1));
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import java.util.Collections;
|
|||
import java.util.Random;
|
||||
|
||||
import org.apache.lucene.analysis.MockAnalyzer;
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.document.FieldType;
|
||||
|
|
@ -96,7 +95,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
|
||||
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
|
||||
// randomized codecs are sometimes too costly for this test:
|
||||
iwc.setCodec(Codec.forName("Lucene86"));
|
||||
iwc.setCodec(TestUtil.getDefaultCodec());
|
||||
iwc.setMergePolicy(newLogMergePolicy());
|
||||
RandomIndexWriter writer= new RandomIndexWriter(random(), directory, iwc);
|
||||
// we'll make a ton of docs, disable store/norms/vectors
|
||||
|
|
@ -141,7 +140,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
iwc = newIndexWriterConfig(new MockAnalyzer(random()));
|
||||
// we need docID order to be preserved:
|
||||
// randomized codecs are sometimes too costly for this test:
|
||||
iwc.setCodec(Codec.forName("Lucene86"));
|
||||
iwc.setCodec(TestUtil.getDefaultCodec());
|
||||
iwc.setMergePolicy(newLogMergePolicy());
|
||||
try (IndexWriter w = new IndexWriter(singleSegmentDirectory, iwc)) {
|
||||
w.forceMerge(1, true);
|
||||
|
|
@ -167,7 +166,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
|
||||
iwc = newIndexWriterConfig(new MockAnalyzer(random()));
|
||||
// randomized codecs are sometimes too costly for this test:
|
||||
iwc.setCodec(Codec.forName("Lucene86"));
|
||||
iwc.setCodec(TestUtil.getDefaultCodec());
|
||||
RandomIndexWriter w = new RandomIndexWriter(random(), dir2, iwc);
|
||||
w.addIndexes(copy);
|
||||
copy.close();
|
||||
|
|
@ -179,7 +178,7 @@ public class TestBoolean2 extends LuceneTestCase {
|
|||
iwc = newIndexWriterConfig(new MockAnalyzer(random()));
|
||||
iwc.setMaxBufferedDocs(TestUtil.nextInt(random(), 50, 1000));
|
||||
// randomized codecs are sometimes too costly for this test:
|
||||
iwc.setCodec(Codec.forName("Lucene86"));
|
||||
iwc.setCodec(TestUtil.getDefaultCodec());
|
||||
RandomIndexWriter w = new RandomIndexWriter(random(), dir2, iwc);
|
||||
|
||||
doc = new Document();
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ package org.apache.lucene.document;
|
|||
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.IndexWriterConfig;
|
||||
|
|
@ -243,7 +242,7 @@ public class TestFloatPointNearestNeighbor extends LuceneTestCase {
|
|||
|
||||
private IndexWriterConfig getIndexWriterConfig() {
|
||||
IndexWriterConfig iwc = newIndexWriterConfig();
|
||||
iwc.setCodec(Codec.forName("Lucene86"));
|
||||
iwc.setCodec(TestUtil.getDefaultCodec());
|
||||
return iwc;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ package org.apache.lucene.search;
|
|||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.document.LatLonDocValuesField;
|
||||
|
|
@ -246,7 +245,7 @@ public class TestNearest extends LuceneTestCase {
|
|||
|
||||
private IndexWriterConfig getIndexWriterConfig() {
|
||||
IndexWriterConfig iwc = newIndexWriterConfig();
|
||||
iwc.setCodec(Codec.forName("Lucene86"));
|
||||
iwc.setCodec(TestUtil.getDefaultCodec());
|
||||
return iwc;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
|
|||
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.PostingsFormat;
|
||||
import org.apache.lucene.codecs.lucene86.Lucene86Codec;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87Codec;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.document.IntPoint;
|
||||
|
|
@ -887,7 +887,7 @@ public class TestSuggestField extends LuceneTestCase {
|
|||
static IndexWriterConfig iwcWithSuggestField(Analyzer analyzer, final Set<String> suggestFields) {
|
||||
IndexWriterConfig iwc = newIndexWriterConfig(random(), analyzer);
|
||||
iwc.setMergePolicy(newLogMergePolicy());
|
||||
Codec filterCodec = new Lucene86Codec() {
|
||||
Codec filterCodec = new Lucene87Codec() {
|
||||
CompletionPostingsFormat.FSTLoadMode fstLoadMode =
|
||||
RandomPicks.randomFrom(random(), CompletionPostingsFormat.FSTLoadMode.values());
|
||||
PostingsFormat postingsFormat = new Completion84PostingsFormat(fstLoadMode);
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ public abstract class CompressingCodec extends FilterCodec {
|
|||
* Create a random instance.
|
||||
*/
|
||||
public static CompressingCodec randomInstance(Random random, int chunkSize, int maxDocsPerChunk, boolean withSegmentSuffix, int blockShift) {
|
||||
switch (random.nextInt(4)) {
|
||||
switch (random.nextInt(5)) {
|
||||
case 0:
|
||||
return new FastCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockShift);
|
||||
case 1:
|
||||
|
|
@ -46,6 +46,8 @@ public abstract class CompressingCodec extends FilterCodec {
|
|||
return new HighCompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockShift);
|
||||
case 3:
|
||||
return new DummyCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockShift);
|
||||
case 4:
|
||||
return new DeflateWithPresetCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockShift);
|
||||
default:
|
||||
throw new AssertionError();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.codecs.compressing;
|
||||
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.DeflateWithPresetDict;
|
||||
|
||||
/** CompressionCodec that uses {@link DeflateWithPresetDict}. */
|
||||
public class DeflateWithPresetCompressingCodec extends CompressingCodec {
|
||||
|
||||
/** Constructor that allows to configure the chunk size. */
|
||||
public DeflateWithPresetCompressingCodec(int chunkSize, int maxDocsPerChunk, boolean withSegmentSuffix, int blockSize) {
|
||||
super("DeflateWithPresetCompressingStoredFieldsData",
|
||||
withSegmentSuffix ? "DeflateWithPresetCompressingStoredFields" : "",
|
||||
new DeflateWithPresetDict(chunkSize/10, chunkSize/3+1), chunkSize, maxDocsPerChunk, blockSize);
|
||||
}
|
||||
|
||||
/** No-arg constructor. */
|
||||
public DeflateWithPresetCompressingCodec() {
|
||||
this(1<<18, 512, false, 10);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -26,6 +26,7 @@ import java.util.Locale;
|
|||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.analysis.MockAnalyzer;
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.FilterCodec;
|
||||
import org.apache.lucene.codecs.PointsFormat;
|
||||
import org.apache.lucene.codecs.PointsReader;
|
||||
|
|
@ -1276,7 +1277,8 @@ public abstract class BaseGeoPointTestCase extends LuceneTestCase {
|
|||
// Else seeds may not reproduce:
|
||||
iwc.setMergeScheduler(new SerialMergeScheduler());
|
||||
int pointsInLeaf = 2 + random().nextInt(4);
|
||||
iwc.setCodec(new FilterCodec("Lucene86", TestUtil.getDefaultCodec()) {
|
||||
final Codec in = TestUtil.getDefaultCodec();
|
||||
iwc.setCodec(new FilterCodec(in.getName(), in) {
|
||||
@Override
|
||||
public PointsFormat pointsFormat() {
|
||||
return new PointsFormat() {
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import java.util.Locale;
|
|||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.analysis.MockAnalyzer;
|
||||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.FilterCodec;
|
||||
import org.apache.lucene.codecs.PointsFormat;
|
||||
import org.apache.lucene.codecs.PointsReader;
|
||||
|
|
@ -1190,7 +1191,8 @@ public abstract class BaseXYPointTestCase extends LuceneTestCase {
|
|||
// Else seeds may not reproduce:
|
||||
iwc.setMergeScheduler(new SerialMergeScheduler());
|
||||
int pointsInLeaf = 2 + random().nextInt(4);
|
||||
iwc.setCodec(new FilterCodec("Lucene86", TestUtil.getDefaultCodec()) {
|
||||
Codec in = TestUtil.getDefaultCodec();
|
||||
iwc.setCodec(new FilterCodec(in.getName(), in) {
|
||||
@Override
|
||||
public PointsFormat pointsFormat() {
|
||||
return new PointsFormat() {
|
||||
|
|
|
|||
|
|
@ -33,8 +33,8 @@ import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat;
|
|||
import org.apache.lucene.codecs.asserting.AssertingPostingsFormat;
|
||||
import org.apache.lucene.codecs.cheapbastard.CheapBastardCodec;
|
||||
import org.apache.lucene.codecs.compressing.CompressingCodec;
|
||||
import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.lucene86.Lucene86Codec;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87Codec;
|
||||
import org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat;
|
||||
import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
|
||||
import org.apache.lucene.index.RandomCodec;
|
||||
|
|
@ -187,8 +187,8 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule {
|
|||
codec = new AssertingCodec();
|
||||
} else if ("Compressing".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 6 && !shouldAvoidCodec("Compressing"))) {
|
||||
codec = CompressingCodec.randomInstance(random);
|
||||
} else if ("Lucene84".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene84"))) {
|
||||
codec = new Lucene86Codec(RandomPicks.randomFrom(random, Lucene50StoredFieldsFormat.Mode.values())
|
||||
} else if ("Lucene87".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene87"))) {
|
||||
codec = new Lucene87Codec(RandomPicks.randomFrom(random, Lucene87StoredFieldsFormat.Mode.values())
|
||||
);
|
||||
} else if (!"random".equals(TEST_CODEC)) {
|
||||
codec = Codec.forName(TEST_CODEC);
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ import org.apache.lucene.codecs.blockterms.LuceneFixedGap;
|
|||
import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
|
||||
import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat;
|
||||
import org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat;
|
||||
import org.apache.lucene.codecs.lucene86.Lucene86Codec;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87Codec;
|
||||
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
|
||||
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
|
||||
import org.apache.lucene.document.BinaryDocValuesField;
|
||||
|
|
@ -919,7 +919,7 @@ public final class TestUtil {
|
|||
* This may be different than {@link Codec#getDefault()} because that is randomized.
|
||||
*/
|
||||
public static Codec getDefaultCodec() {
|
||||
return new Lucene86Codec();
|
||||
return new Lucene87Codec();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
org.apache.lucene.codecs.asserting.AssertingCodec
|
||||
org.apache.lucene.codecs.cheapbastard.CheapBastardCodec
|
||||
org.apache.lucene.codecs.compressing.DeflateWithPresetCompressingCodec
|
||||
org.apache.lucene.codecs.compressing.FastCompressingCodec
|
||||
org.apache.lucene.codecs.compressing.FastDecompressionCompressingCodec
|
||||
org.apache.lucene.codecs.compressing.HighCompressionCompressingCodec
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@ import java.util.Locale;
|
|||
import org.apache.lucene.codecs.Codec;
|
||||
import org.apache.lucene.codecs.DocValuesFormat;
|
||||
import org.apache.lucene.codecs.PostingsFormat;
|
||||
import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
|
||||
import org.apache.lucene.codecs.lucene86.Lucene86Codec;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.Mode;
|
||||
import org.apache.lucene.codecs.lucene87.Lucene87Codec;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
|
|
@ -92,7 +92,7 @@ public class SchemaCodecFactory extends CodecFactory implements SolrCoreAware {
|
|||
compressionMode = SOLR_DEFAULT_COMPRESSION_MODE;
|
||||
log.debug("Using default compressionMode: {}", compressionMode);
|
||||
}
|
||||
codec = new Lucene86Codec(compressionMode) {
|
||||
codec = new Lucene87Codec(compressionMode) {
|
||||
@Override
|
||||
public PostingsFormat getPostingsFormatForField(String field) {
|
||||
final SchemaField schemaField = core.getLatestSchema().getFieldOrNull(field);
|
||||
|
|
|
|||
Loading…
Reference in New Issue