Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-9917: Smaller block sizes for BEST_SPEED. (#257) 

This reduces the block size for BEST_SPEED in order to trade some compression
ratio in exchange for better retrieval speed.
This commit is contained in:
Adrien Grand 2021-08-26 15:04:51 +02:00 committed by GitHub
parent f6e3b08ae9
commit f1fdd2465c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -425,6 +425,9 @@ Improvements
* LUCENE-5309: Optimize facet counting for single-valued SSDV / StringValueFacetCounts. (Greg Miller)
* LUCENE-9917: The BEST_SPEED compression mode now trades more compression ratio
in exchange of faster reads. (Adrien Grand)
Optimizations
---------------------
* LUCENE-9996: Improved memory efficiency of IndexWriter's RAM buffer, in

4
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/LZ4WithPresetDictCompressionMode.java
View File

@ -38,8 +38,8 @@ public final class LZ4WithPresetDictCompressionMode extends CompressionMode {
// Shoot for 10 sub blocks
private static final int NUM_SUB_BLOCKS = 10;
// And a dictionary whose size is about 16x smaller than sub blocks
private static final int DICT_SIZE_FACTOR = 16;
// And a dictionary whose size is about 2x smaller than sub blocks
private static final int DICT_SIZE_FACTOR = 2;
/** Sole constructor. */
public LZ4WithPresetDictCompressionMode() {}

4
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87StoredFieldsFormat.java
View File

@ -167,9 +167,9 @@ public class Lucene87StoredFieldsFormat extends StoredFieldsFormat {
public static final CompressionMode BEST_COMPRESSION_MODE =
new DeflateWithPresetDictCompressionMode();
// Shoot for 10 sub blocks of 60kB each.
// Shoot for 10 sub blocks of 8kB each.
/** Block length for {@link Mode#BEST_SPEED} */
protected static final int BEST_SPEED_BLOCK_LENGTH = 10 * 60 * 1024;
protected static final int BEST_SPEED_BLOCK_LENGTH = 10 * 8 * 1024;
/** Compression mode for {@link Mode#BEST_SPEED} */
public static final CompressionMode BEST_SPEED_MODE = new LZ4WithPresetDictCompressionMode();

4
lucene/core/src/java/org/apache/lucene/codecs/lucene90/LZ4WithPresetDictCompressionMode.java
View File

@ -38,8 +38,8 @@ public final class LZ4WithPresetDictCompressionMode extends CompressionMode {
// Shoot for 10 sub blocks
private static final int NUM_SUB_BLOCKS = 10;
// And a dictionary whose size is about 16x smaller than sub blocks
private static final int DICT_SIZE_FACTOR = 16;
// And a dictionary whose size is about 2x smaller than sub blocks
private static final int DICT_SIZE_FACTOR = 2;
/** Sole constructor. */
public LZ4WithPresetDictCompressionMode() {}

4
lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90StoredFieldsFormat.java
View File

@ -178,8 +178,8 @@ public class Lucene90StoredFieldsFormat extends StoredFieldsFormat {
public static final CompressionMode BEST_COMPRESSION_MODE =
new DeflateWithPresetDictCompressionMode();
// Shoot for 10 sub blocks of 60kB each.
private static final int BEST_SPEED_BLOCK_LENGTH = 10 * 60 * 1024;
// Shoot for 10 sub blocks of 8kB each.
private static final int BEST_SPEED_BLOCK_LENGTH = 10 * 8 * 1024;
/** Compression mode for {@link Mode#BEST_SPEED} */
public static final CompressionMode BEST_SPEED_MODE = new LZ4WithPresetDictCompressionMode();