231 - 214) bytes.
* @lucene.experimental
*/
-public final class Lucene50StoredFieldsFormat extends StoredFieldsFormat {
+public class Lucene50StoredFieldsFormat extends StoredFieldsFormat {
/** Configuration option for stored fields. */
public static enum Mode {
@@ -126,7 +126,7 @@ public final class Lucene50StoredFieldsFormat extends StoredFieldsFormat {
}
@Override
- public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
+ public final StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
String value = si.getAttribute(MODE_KEY);
if (value == null) {
throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);
@@ -137,12 +137,7 @@ public final class Lucene50StoredFieldsFormat extends StoredFieldsFormat {
@Override
public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException {
- String previous = si.putAttribute(MODE_KEY, mode.name());
- if (previous != null && previous.equals(mode.name()) == false) {
- throw new IllegalStateException("found existing value for " + MODE_KEY + " for segment: " + si.name +
- "old=" + previous + ", new=" + mode.name());
- }
- return impl(mode).fieldsWriter(directory, si, context);
+ throw new UnsupportedOperationException("Old codecs may only be used for reading");
}
StoredFieldsFormat impl(Mode mode) {
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene84/Lucene84Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene84/Lucene84Codec.java
index bef563301ba..90918c163d2 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene84/Lucene84Codec.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene84/Lucene84Codec.java
@@ -97,7 +97,7 @@ public class Lucene84Codec extends Codec {
}
@Override
- public final StoredFieldsFormat storedFieldsFormat() {
+ public StoredFieldsFormat storedFieldsFormat() {
return storedFieldsFormat;
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene86/Lucene86Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene86/Lucene86Codec.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene86/Lucene86Codec.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene86/Lucene86Codec.java
index 3f69874ef20..e2974655e75 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene86/Lucene86Codec.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene86/Lucene86Codec.java
@@ -97,7 +97,7 @@ public class Lucene86Codec extends Codec {
}
@Override
- public final StoredFieldsFormat storedFieldsFormat() {
+ public StoredFieldsFormat storedFieldsFormat() {
return storedFieldsFormat;
}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene86/package.html b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene86/package.html
new file mode 100644
index 00000000000..10560c624c2
--- /dev/null
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene86/package.html
@@ -0,0 +1,25 @@
+
+
+
+
+
+
+
+Lucene 8.6 file format.
+
+
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
index cf7a945e133..d6732336efd 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -15,3 +15,4 @@
org.apache.lucene.codecs.lucene80.Lucene80Codec
org.apache.lucene.codecs.lucene84.Lucene84Codec
+org.apache.lucene.codecs.lucene86.Lucene86Codec
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/Lucene50RWStoredFieldsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/Lucene50RWStoredFieldsFormat.java
new file mode 100644
index 00000000000..82d1c96cf25
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/Lucene50RWStoredFieldsFormat.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene50;
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.StoredFieldsWriter;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+
+/**
+ * RW impersonation of Lucene50StoredFieldsFormat.
+ */
+public final class Lucene50RWStoredFieldsFormat extends Lucene50StoredFieldsFormat {
+
+ /** No-argument constructor. */
+ public Lucene50RWStoredFieldsFormat() {
+ super();
+ }
+
+ /** Constructor that takes a mode. */
+ public Lucene50RWStoredFieldsFormat(Lucene50StoredFieldsFormat.Mode mode) {
+ super(mode);
+ }
+
+ @Override
+ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException {
+ String previous = si.putAttribute(MODE_KEY, mode.name());
+ if (previous != null && previous.equals(mode.name()) == false) {
+ throw new IllegalStateException("found existing value for " + MODE_KEY + " for segment: " + si.name +
+ "old=" + previous + ", new=" + mode.name());
+ }
+ return impl(mode).fieldsWriter(directory, si, context);
+ }
+
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormat.java
similarity index 92%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormat.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormat.java
index 4c7bed478c0..fec9e43f4e2 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormat.java
@@ -18,12 +18,12 @@ package org.apache.lucene.codecs.lucene50;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene86.Lucene86RWCodec;
import org.apache.lucene.index.BaseStoredFieldsFormatTestCase;
-import org.apache.lucene.util.TestUtil;
public class TestLucene50StoredFieldsFormat extends BaseStoredFieldsFormatTestCase {
@Override
protected Codec getCodec() {
- return TestUtil.getDefaultCodec();
+ return new Lucene86RWCodec();
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
similarity index 91%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
index cccee736d46..41b4b845428 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
@@ -20,7 +20,7 @@ package org.apache.lucene.codecs.lucene50;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
-import org.apache.lucene.codecs.lucene86.Lucene86Codec;
+import org.apache.lucene.codecs.lucene86.Lucene86RWCodec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.BaseStoredFieldsFormatTestCase;
@@ -32,7 +32,7 @@ import org.apache.lucene.store.Directory;
public class TestLucene50StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {
@Override
protected Codec getCodec() {
- return new Lucene86Codec(Mode.BEST_COMPRESSION);
+ return new Lucene86RWCodec(Mode.BEST_COMPRESSION);
}
/**
@@ -43,7 +43,7 @@ public class TestLucene50StoredFieldsFormatHighCompression extends BaseStoredFie
Directory dir = newDirectory();
for (int i = 0; i < 10; i++) {
IndexWriterConfig iwc = newIndexWriterConfig();
- iwc.setCodec(new Lucene86Codec(RandomPicks.randomFrom(random(), Mode.values())));
+ iwc.setCodec(new Lucene86RWCodec(RandomPicks.randomFrom(random(), Mode.values())));
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
Document doc = new Document();
doc.add(new StoredField("field1", "value1"));
@@ -70,7 +70,7 @@ public class TestLucene50StoredFieldsFormatHighCompression extends BaseStoredFie
public void testInvalidOptions() {
expectThrows(NullPointerException.class, () -> {
- new Lucene86Codec(null);
+ new Lucene86RWCodec(null);
});
expectThrows(NullPointerException.class, () -> {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java
similarity index 100%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene70/TestLucene70SegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene70/TestLucene70SegmentInfoFormat.java
index ac516a121ef..d9dd0198a51 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene70/TestLucene70SegmentInfoFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene70/TestLucene70SegmentInfoFormat.java
@@ -18,8 +18,7 @@
package org.apache.lucene.codecs.lucene70;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.lucene84.Lucene84RWCodec;
import org.apache.lucene.index.BaseSegmentInfoFormatTestCase;
import org.apache.lucene.util.Version;
@@ -32,11 +31,6 @@ public class TestLucene70SegmentInfoFormat extends BaseSegmentInfoFormatTestCase
@Override
protected Codec getCodec() {
- return new FilterCodec("Lucene84", Codec.forName("Lucene84")) {
- @Override
- public SegmentInfoFormat segmentInfoFormat() {
- return new Lucene70RWSegmentInfoFormat();
- }
- };
+ return new Lucene84RWCodec();
}
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene84/Lucene84RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene84/Lucene84RWCodec.java
index c1fd4677f92..0f74e79f508 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene84/Lucene84RWCodec.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene84/Lucene84RWCodec.java
@@ -18,6 +18,8 @@ package org.apache.lucene.codecs.lucene84;
import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.lucene50.Lucene50RWStoredFieldsFormat;
import org.apache.lucene.codecs.lucene60.Lucene60RWPointsFormat;
import org.apache.lucene.codecs.lucene70.Lucene70RWSegmentInfoFormat;
@@ -36,4 +38,9 @@ public class Lucene84RWCodec extends Lucene84Codec {
return new Lucene70RWSegmentInfoFormat();
}
+ @Override
+ public StoredFieldsFormat storedFieldsFormat() {
+ return new Lucene50RWStoredFieldsFormat();
+ }
+
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene86/Lucene86RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene86/Lucene86RWCodec.java
new file mode 100644
index 00000000000..72e2bee7e07
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene86/Lucene86RWCodec.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene86;
+
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.lucene50.Lucene50RWStoredFieldsFormat;
+import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat;
+
+/**
+ * RW impersonation of {@link Lucene86Codec}.
+ */
+public class Lucene86RWCodec extends Lucene86Codec {
+
+ private final StoredFieldsFormat storedFieldsFormat;
+
+ /** No arguments constructor. */
+ public Lucene86RWCodec() {
+ storedFieldsFormat = new Lucene50RWStoredFieldsFormat();
+ }
+
+ /** Constructor that takes a mode. */
+ public Lucene86RWCodec(Lucene50StoredFieldsFormat.Mode mode) {
+ storedFieldsFormat = new Lucene50RWStoredFieldsFormat(mode);
+ }
+
+ @Override
+ public StoredFieldsFormat storedFieldsFormat() {
+ return storedFieldsFormat;
+ }
+
+}
diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
index db64781cff7..e44b046aa29 100644
--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
@@ -29,7 +29,7 @@ import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene86.Lucene86Codec;
+import org.apache.lucene.codecs.lucene87.Lucene87Codec;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexDeletionPolicy;
@@ -138,7 +138,7 @@ public class CreateIndexTask extends PerfTask {
if (defaultCodec == null && postingsFormat != null) {
try {
final PostingsFormat postingsFormatChosen = PostingsFormat.forName(postingsFormat);
- iwConf.setCodec(new Lucene86Codec() {
+ iwConf.setCodec(new Lucene87Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return postingsFormatChosen;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
index 8b5ca14ff89..14fa7935f9f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
@@ -57,7 +57,7 @@ public abstract class Codec implements NamedSPILoader.NamedSPI {
}
// TODO: should we use this, or maybe a system property is better?
- static Codec defaultCodec = LOADER.lookup("Lucene86");
+ static Codec defaultCodec = LOADER.lookup("Lucene87");
}
private final String name;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java
index 00412d5473c..9b65fb4fe6d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java
@@ -20,6 +20,7 @@ package org.apache.lucene.codecs.lucene50;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.compressing.FieldsIndexWriter;
+import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat;
import org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat;
import org.apache.lucene.codecs.compressing.CompressionMode;
import org.apache.lucene.store.DataOutput;
@@ -29,7 +30,7 @@ import org.apache.lucene.util.packed.PackedInts;
/**
* Lucene 5.0 {@link TermVectorsFormat term vectors format}.
* - * Very similarly to {@link Lucene50StoredFieldsFormat}, this format is based + * Very similarly to {@link Lucene87StoredFieldsFormat}, this format is based * on compressed chunks of data, with document-level granularity so that a * document can never span across distinct chunks. Moreover, data is made as * compact as possible:
+ * If you want to reuse functionality of this codec in another codec, extend
+ * {@link FilterCodec}.
+ *
+ * @see org.apache.lucene.codecs.lucene86 package documentation for file format details.
+ *
+ * @lucene.experimental
+ */
+public class Lucene87Codec extends Codec {
+ private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
+ private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat();
+ private final SegmentInfoFormat segmentInfosFormat = new Lucene86SegmentInfoFormat();
+ private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
+ private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
+ private final PointsFormat pointsFormat = new Lucene86PointsFormat();
+ private final PostingsFormat defaultFormat;
+
+ private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
+ @Override
+ public PostingsFormat getPostingsFormatForField(String field) {
+ return Lucene87Codec.this.getPostingsFormatForField(field);
+ }
+ };
+
+ private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
+ @Override
+ public DocValuesFormat getDocValuesFormatForField(String field) {
+ return Lucene87Codec.this.getDocValuesFormatForField(field);
+ }
+ };
+
+ private final StoredFieldsFormat storedFieldsFormat;
+
+ /**
+ * Instantiates a new codec.
+ */
+ public Lucene87Codec() {
+ this(Lucene87StoredFieldsFormat.Mode.BEST_SPEED);
+ }
+
+ /**
+ * Instantiates a new codec, specifying the stored fields compression
+ * mode to use.
+ * @param mode stored fields compression mode to use for newly
+ * flushed/merged segments.
+ */
+ public Lucene87Codec(Lucene87StoredFieldsFormat.Mode mode) {
+ super("Lucene87");
+ this.storedFieldsFormat = new Lucene87StoredFieldsFormat(Objects.requireNonNull(mode));
+ this.defaultFormat = new Lucene84PostingsFormat();
+ }
+
+ @Override
+ public final StoredFieldsFormat storedFieldsFormat() {
+ return storedFieldsFormat;
+ }
+
+ @Override
+ public final TermVectorsFormat termVectorsFormat() {
+ return vectorsFormat;
+ }
+
+ @Override
+ public final PostingsFormat postingsFormat() {
+ return postingsFormat;
+ }
+
+ @Override
+ public final FieldInfosFormat fieldInfosFormat() {
+ return fieldInfosFormat;
+ }
+
+ @Override
+ public final SegmentInfoFormat segmentInfoFormat() {
+ return segmentInfosFormat;
+ }
+
+ @Override
+ public final LiveDocsFormat liveDocsFormat() {
+ return liveDocsFormat;
+ }
+
+ @Override
+ public final CompoundFormat compoundFormat() {
+ return compoundFormat;
+ }
+
+ @Override
+ public final PointsFormat pointsFormat() {
+ return pointsFormat;
+ }
+
+ /** Returns the postings format that should be used for writing
+ * new segments of field.
+ *
+ * The default implementation always returns "Lucene84".
+ *
+ * WARNING: if you subclass, you are responsible for index
+ * backwards compatibility: future version of Lucene are only
+ * guaranteed to be able to read the default implementation.
+ */
+ public PostingsFormat getPostingsFormatForField(String field) {
+ return defaultFormat;
+ }
+
+ /** Returns the docvalues format that should be used for writing
+ * new segments of field.
+ *
+ * The default implementation always returns "Lucene80".
+ *
+ * WARNING: if you subclass, you are responsible for index + * backwards compatibility: future version of Lucene are only + * guaranteed to be able to read the default implementation. + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80"); + + private final NormsFormat normsFormat = new Lucene80NormsFormat(); + + @Override + public final NormsFormat normsFormat() { + return normsFormat; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87StoredFieldsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87StoredFieldsFormat.java new file mode 100644 index 00000000000..c2bbcedfc74 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87StoredFieldsFormat.java @@ -0,0 +1,378 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs.lucene87; + +import java.io.IOException; +import java.util.Objects; +import java.util.zip.DataFormatException; +import java.util.zip.Deflater; +import java.util.zip.Inflater; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.compressing.Compressor; +import org.apache.lucene.codecs.compressing.Decompressor; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.packed.DirectMonotonicWriter; + +/** + * Lucene 8.7 stored fields format. + * + *
Principle + *
This {@link StoredFieldsFormat} compresses blocks of documents in + * order to improve the compression ratio compared to document-level + * compression. It uses the LZ4 + * compression algorithm by default in 16KB blocks, which is fast to compress + * and very fast to decompress data. Although the default compression method + * that is used ({@link Mode#BEST_SPEED BEST_SPEED}) focuses more on speed than on + * compression ratio, it should provide interesting compression ratios + * for redundant inputs (such as log files, HTML or plain text). For higher + * compression, you can choose ({@link Mode#BEST_COMPRESSION BEST_COMPRESSION}), + * which uses the DEFLATE + * algorithm with 48kB blocks and shared dictionaries for a better ratio at the + * expense of slower performance. These two options can be configured like this: + *
+ * // the default: for high performance + * indexWriterConfig.setCodec(new Lucene87Codec(Mode.BEST_SPEED)); + * // instead for higher performance (but slower): + * // indexWriterConfig.setCodec(new Lucene87Codec(Mode.BEST_COMPRESSION)); + *+ *
File formats + *
Stored fields are represented by three files: + *
A fields data file (extension .fdt). This file stores a compact
+ * representation of documents in compressed blocks of 16KB or more. When
+ * writing a segment, documents are appended to an in-memory byte[]
+ * buffer. When its size reaches 16KB or more, some metadata about the documents
+ * is flushed to disk, immediately followed by a compressed representation of
+ * the buffer using the
+ * LZ4
+ * compression format.
Notes + *
A fields index file (extension .fdx). This file stores two
+ * {@link DirectMonotonicWriter monotonic arrays}, one for the first doc IDs of
+ * each block of compressed documents, and another one for the corresponding
+ * offsets on disk. At search time, the array containing doc IDs is
+ * binary-searched in order to find the block that contains the expected doc ID,
+ * and the associated offset on disk is retrieved from the second array.
A fields meta file (extension .fdm). This file stores metadata
+ * about the monotonic arrays stored in the index file.
Known limitations + *
This {@link StoredFieldsFormat} does not support individual documents
+ * larger than (231 - 214) bytes.
+ * @lucene.experimental
+ */
+public class Lucene87StoredFieldsFormat extends StoredFieldsFormat {
+
+ /** Configuration option for stored fields. */
+ public static enum Mode {
+ /** Trade compression ratio for retrieval speed. */
+ BEST_SPEED,
+ /** Trade retrieval speed for compression ratio. */
+ BEST_COMPRESSION
+ }
+
+ /** Attribute key for compression mode. */
+ public static final String MODE_KEY = Lucene87StoredFieldsFormat.class.getSimpleName() + ".mode";
+
+ final Mode mode;
+
+ /** Stored fields format with default options */
+ public Lucene87StoredFieldsFormat() {
+ this(Mode.BEST_SPEED);
+ }
+
+ /** Stored fields format with specified mode */
+ public Lucene87StoredFieldsFormat(Mode mode) {
+ this.mode = Objects.requireNonNull(mode);
+ }
+
+ @Override
+ public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
+ String value = si.getAttribute(MODE_KEY);
+ if (value == null) {
+ throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);
+ }
+ Mode mode = Mode.valueOf(value);
+ return impl(mode).fieldsReader(directory, si, fn, context);
+ }
+
+ @Override
+ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException {
+ String previous = si.putAttribute(MODE_KEY, mode.name());
+ if (previous != null && previous.equals(mode.name()) == false) {
+ throw new IllegalStateException("found existing value for " + MODE_KEY + " for segment: " + si.name +
+ "old=" + previous + ", new=" + mode.name());
+ }
+ return impl(mode).fieldsWriter(directory, si, context);
+ }
+
+ StoredFieldsFormat impl(Mode mode) {
+ switch (mode) {
+ case BEST_SPEED:
+ return new CompressingStoredFieldsFormat("Lucene87StoredFieldsFastData", CompressionMode.FAST, 16*1024, 128, 10);
+ case BEST_COMPRESSION:
+ return new CompressingStoredFieldsFormat("Lucene87StoredFieldsHighData", BEST_COMPRESSION_MODE, BEST_COMPRESSION_BLOCK_LENGTH, 512, 10);
+ default: throw new AssertionError();
+ }
+ }
+
+ // 8kB seems to be a good trade-off between higher compression rates by not
+ // having to fully bootstrap a dictionary, and indexing rate by not spending
+ // too much CPU initializing data-structures to find strings in this preset
+ // dictionary.
+ private static final int BEST_COMPRESSION_DICT_LENGTH = 8 * 1024;
+ // 48kB seems like a nice trade-off because it's small enough to keep
+ // retrieval fast, yet sub blocks can find strings in a window of 26kB of
+ // data on average (the window grows from 8kB to 32kB in the first 24kB, and
+ // then DEFLATE can use 32kB for the last 24kB) which is close enough to the
+ // maximum window length of DEFLATE of 32kB.
+ private static final int BEST_COMPRESSION_SUB_BLOCK_LENGTH = 48 * 1024;
+ // We shoot for 10 sub blocks per block, which should hopefully amortize the
+ // space overhead of having the first 8kB compressed without any preset dict,
+ // and then remove 8kB in order to avoid creating a tiny 11th sub block if
+ // documents are small.
+ private static final int BEST_COMPRESSION_BLOCK_LENGTH = BEST_COMPRESSION_DICT_LENGTH + 10 * BEST_COMPRESSION_SUB_BLOCK_LENGTH - 8 * 1024;
+
+ /** Compression mode for {@link Mode#BEST_COMPRESSION} */
+ public static final DeflateWithPresetDict BEST_COMPRESSION_MODE = new DeflateWithPresetDict(BEST_COMPRESSION_DICT_LENGTH, BEST_COMPRESSION_SUB_BLOCK_LENGTH);
+
+ /**
+ * A compression mode that trades speed for compression ratio. Although
+ * compression and decompression might be slow, this compression mode should
+ * provide a good compression ratio. This mode might be interesting if/when
+ * your index size is much bigger than your OS cache.
+ */
+ public static class DeflateWithPresetDict extends CompressionMode {
+
+ private final int dictLength, subBlockLength;
+
+ /** Sole constructor. */
+ public DeflateWithPresetDict(int dictLength, int subBlockLength) {
+ this.dictLength = dictLength;
+ this.subBlockLength = subBlockLength;
+ }
+
+ @Override
+ public Compressor newCompressor() {
+ // notes:
+ // 3 is the highest level that doesn't have lazy match evaluation
+ // 6 is the default, higher than that is just a waste of cpu
+ return new DeflateWithPresetDictCompressor(6, dictLength, subBlockLength);
+ }
+
+ @Override
+ public Decompressor newDecompressor() {
+ return new DeflateWithPresetDictDecompressor();
+ }
+
+ @Override
+ public String toString() {
+ return "BEST_COMPRESSION";
+ }
+
+ };
+
+ private static final class DeflateWithPresetDictDecompressor extends Decompressor {
+
+ byte[] compressed;
+
+ DeflateWithPresetDictDecompressor() {
+ compressed = new byte[0];
+ }
+
+ private void doDecompress(DataInput in, Inflater decompressor, BytesRef bytes) throws IOException {
+ final int compressedLength = in.readVInt();
+ if (compressedLength == 0) {
+ return;
+ }
+ // pad with extra "dummy byte": see javadocs for using Inflater(true)
+ // we do it for compliance, but it's unnecessary for years in zlib.
+ final int paddedLength = compressedLength + 1;
+ compressed = ArrayUtil.grow(compressed, paddedLength);
+ in.readBytes(compressed, 0, compressedLength);
+ compressed[compressedLength] = 0; // explicitly set dummy byte to 0
+
+ // extra "dummy byte"
+ decompressor.setInput(compressed, 0, paddedLength);
+ try {
+ bytes.length += decompressor.inflate(bytes.bytes, bytes.length, bytes.bytes.length - bytes.length);
+ } catch (DataFormatException e) {
+ throw new IOException(e);
+ }
+ if (decompressor.finished() == false) {
+ throw new CorruptIndexException("Invalid decoder state: needsInput=" + decompressor.needsInput()
+ + ", needsDict=" + decompressor.needsDictionary(), in);
+ }
+ }
+
+ @Override
+ public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException {
+ assert offset + length <= originalLength;
+ if (length == 0) {
+ bytes.length = 0;
+ return;
+ }
+ final int dictLength = in.readVInt();
+ final int blockLength = in.readVInt();
+ bytes.bytes = ArrayUtil.grow(bytes.bytes, dictLength);
+ bytes.offset = bytes.length = 0;
+
+ final Inflater decompressor = new Inflater(true);
+ try {
+ // Read the dictionary
+ doDecompress(in, decompressor, bytes);
+ if (dictLength != bytes.length) {
+ throw new CorruptIndexException("Unexpected dict length", in);
+ }
+
+ int offsetInBlock = dictLength;
+ int offsetInBytesRef = offset;
+
+ // Skip unneeded blocks
+ while (offsetInBlock + blockLength < offset) {
+ final int compressedLength = in.readVInt();
+ in.skipBytes(compressedLength);
+ offsetInBlock += blockLength;
+ offsetInBytesRef -= blockLength;
+ }
+
+ // Read blocks that intersect with the interval we need
+ while (offsetInBlock < offset + length) {
+ bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + blockLength);
+ decompressor.reset();
+ decompressor.setDictionary(bytes.bytes, 0, dictLength);
+ doDecompress(in, decompressor, bytes);
+ offsetInBlock += blockLength;
+ }
+
+ bytes.offset = offsetInBytesRef;
+ bytes.length = length;
+ assert bytes.isValid();
+ } finally {
+ decompressor.end();
+ }
+ }
+
+ @Override
+ public Decompressor clone() {
+ return new DeflateWithPresetDictDecompressor();
+ }
+
+ }
+
+ private static class DeflateWithPresetDictCompressor extends Compressor {
+
+ final int dictLength;
+ final int blockLength;
+ final Deflater compressor;
+ byte[] compressed;
+ boolean closed;
+
+ DeflateWithPresetDictCompressor(int level, int dictLength, int blockLength) {
+ compressor = new Deflater(level, true);
+ compressed = new byte[64];
+ this.dictLength = dictLength;
+ this.blockLength = blockLength;
+ }
+
+ private void doCompress(byte[] bytes, int off, int len, DataOutput out) throws IOException {
+ if (len == 0) {
+ out.writeVInt(0);
+ return;
+ }
+ compressor.setInput(bytes, off, len);
+ compressor.finish();
+ if (compressor.needsInput()) {
+ throw new IllegalStateException();
+ }
+
+ int totalCount = 0;
+ for (;;) {
+ final int count = compressor.deflate(compressed, totalCount, compressed.length - totalCount);
+ totalCount += count;
+ assert totalCount <= compressed.length;
+ if (compressor.finished()) {
+ break;
+ } else {
+ compressed = ArrayUtil.grow(compressed);
+ }
+ }
+
+ out.writeVInt(totalCount);
+ out.writeBytes(compressed, totalCount);
+ }
+
+ @Override
+ public void compress(byte[] bytes, int off, int len, DataOutput out) throws IOException {
+ final int dictLength = Math.min(this.dictLength, len);
+ out.writeVInt(dictLength);
+ out.writeVInt(blockLength);
+ final int end = off + len;
+
+ // Compress the dictionary first
+ compressor.reset();
+ doCompress(bytes, off, dictLength, out);
+
+ // And then sub blocks
+ for (int start = off + dictLength; start < end; start += blockLength) {
+ compressor.reset();
+ compressor.setDictionary(bytes, off, dictLength);
+ doCompress(bytes, start, Math.min(blockLength, off + len - start), out);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed == false) {
+ compressor.end();
+ closed = true;
+ }
+ }
+ }
+
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene87/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene87/package-info.java
new file mode 100644
index 00000000000..75facdb2fb4
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene87/package-info.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Lucene 8.7 file format.
+ *
+ *
This document defines the index file formats used in this version of Lucene.
+ * If you are using a different version of Lucene, please consult the copy of
+ * docs/ that was distributed with
+ * the version you are using.
This document attempts to provide a high-level definition of the Apache + * Lucene file formats.
+ *The fundamental concepts in Lucene are index, document, field and term.
+ *An index contains a sequence of documents.
+ *The same sequence of bytes in two different fields is considered a different + * term. Thus terms are represented as a pair: the string naming the field, and the + * bytes within the field.
+ * + *The index stores statistics about terms in order to make term-based search + * more efficient. Lucene's index falls into the family of indexes known as an + * inverted index. This is because it can list, for a term, the documents + * that contain it. This is the inverse of the natural relationship, in which + * documents list terms.
+ * + *In Lucene, fields may be stored, in which case their text is stored + * in the index literally, in a non-inverted manner. Fields that are inverted are + * called indexed. A field may be both stored and indexed.
+ *The text of a field may be tokenized into terms to be indexed, or the + * text of a field may be used literally as a term to be indexed. Most fields are + * tokenized, but sometimes it is useful for certain identifier fields to be + * indexed literally.
+ *See the {@link org.apache.lucene.document.Field Field} + * java docs for more information on Fields.
+ * + *Lucene indexes may be composed of multiple sub-indexes, or segments. + * Each segment is a fully independent index, which could be searched separately. + * Indexes evolve by:
+ *Searches may involve multiple segments and/or multiple indexes, each index + * potentially composed of a set of segments.
+ * + *Internally, Lucene refers to documents by an integer document number. + * The first document added to an index is numbered zero, and each subsequent + * document added gets a number one greater than the previous.
+ *Note that a document's number may change, so caution should be taken when + * storing these numbers outside of Lucene. In particular, numbers may change in + * the following situations:
+ *The numbers stored in each segment are unique only within the segment, and + * must be converted before they can be used in a larger context. The standard + * technique is to allocate each segment a range of values, based on the range of + * numbers used in that segment. To convert a document number from a segment to an + * external value, the segment's base document number is added. To convert + * an external value back to a segment-specific value, the segment is identified + * by the range that the external value is in, and the segment's base value is + * subtracted. For example two five document segments might be combined, so that + * the first segment has a base value of zero, and the second of five. Document + * three from the second segment would have an external value of eight.
+ *When documents are deleted, gaps are created in the numbering. These are + * eventually removed as the index evolves through merging. Deleted documents are + * dropped when segments are merged. A freshly-merged segment thus has no gaps in + * its numbering.
+ *Each segment index maintains the following:
+ *Details on each of these are provided in their linked pages.
+ *All files belonging to a segment have the same name with varying extensions. + * The extensions correspond to the different file formats described below. When + * using the Compound File format (default for small segments) these files (except + * for the Segment info file, the Lock file, and Deleted documents file) are collapsed + * into a single .cfs file (see below for details)
+ *Typically, all segments in an index are stored in a single directory, + * although this is not required.
+ *File names are never re-used. That is, when any file is saved + * to the Directory it is given a never before used filename. This is achieved + * using a simple generations approach. For example, the first segments file is + * segments_1, then segments_2, etc. The generation is a sequential long integer + * represented in alpha-numeric (base 36) form.
+ *The following table summarizes the names and extensions of the files in + * Lucene:
+ *| Name | + *Extension | + *Brief Description | + *
|---|---|---|
| {@link org.apache.lucene.index.SegmentInfos Segments File} | + *segments_N | + *Stores information about a commit point | + *
| Lock File | + *write.lock | + *The Write lock prevents multiple IndexWriters from writing to the same + * file. | + *
| {@link org.apache.lucene.codecs.lucene86.Lucene86SegmentInfoFormat Segment Info} | + *.si | + *Stores metadata about a segment | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat Compound File} | + *.cfs, .cfe | + *An optional "virtual" file consisting of all the other index files for + * systems that frequently run out of file handles. | + *
| {@link org.apache.lucene.codecs.lucene60.Lucene60FieldInfosFormat Fields} | + *.fnm | + *Stores information about the fields | + *
| {@link org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat Field Index} | + *.fdx | + *Contains pointers to field data | + *
| {@link org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat Field Data} | + *.fdt | + *The stored fields for documents | + *
| {@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Term Dictionary} | + *.tim | + *The term dictionary, stores term info | + *
| {@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Term Index} | + *.tip | + *The index into the Term Dictionary | + *
| {@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Frequencies} | + *.doc | + *Contains the list of docs which contain each term along with frequency | + *
| {@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Positions} | + *.pos | + *Stores position information about where a term occurs in the index | + *
| {@link org.apache.lucene.codecs.lucene84.Lucene84PostingsFormat Payloads} | + *.pay | + *Stores additional per-position metadata information such as character offsets and user payloads | + *
| {@link org.apache.lucene.codecs.lucene80.Lucene80NormsFormat Norms} | + *.nvd, .nvm | + *Encodes length and boost factors for docs and fields | + *
| {@link org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat Per-Document Values} | + *.dvd, .dvm | + *Encodes additional scoring factors or other per-document information. | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat Term Vector Index} | + *.tvx | + *Stores offset into the document data file | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat Term Vector Data} | + *.tvd | + *Contains term vector data. | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50LiveDocsFormat Live Documents} | + *.liv | + *Info about what documents are live | + *
| {@link org.apache.lucene.codecs.lucene86.Lucene86PointsFormat Point values} | + *.dii, .dim | + *Holds indexed points, if any | + *
Compatibility notes are provided in this document, describing how file + * formats have changed from prior versions:
+ *Lucene uses a Java int to refer to
+ * document numbers, and the index file format uses an Int32
+ * on-disk to store document numbers. This is a limitation
+ * of both the index file format and the current implementation. Eventually these
+ * should be replaced with either UInt64 values, or
+ * better yet, {@link org.apache.lucene.store.DataOutput#writeVInt VInt} values which have no limit.