Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5161: set sane default readChunkSizes, make the setter work, and test the chunking 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1512723 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2013-08-10 14:24:42 +00:00
parent 0e332a923d
commit 51314afdd4
3 changed files with 11 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/CHANGES.txt
View File

@ -112,6 +112,10 @@ Bug Fixes
seek/lookup which can cause sideeffects if done on a cached FST root arc. seek/lookup which can cause sideeffects if done on a cached FST root arc.
(Simon Willnauer) (Simon Willnauer)
* LUCENE-5161: Fix default chunk sizes in FSDirectory.java to not be unnecessarily large,
and fix setReadChunkSize to always work regardless of whether the machine is 32bit
or 64bit. (Uwe Schindler, Robert Muir)
API Changes API Changes
* LUCENE-5094: Add ramBytesUsed() to MultiDocValues.OrdinalMap. * LUCENE-5094: Add ramBytesUsed() to MultiDocValues.OrdinalMap.

16
lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
View File

@ -112,12 +112,9 @@ import org.apache.lucene.util.Constants;
public abstract class FSDirectory extends Directory { public abstract class FSDirectory extends Directory {
/** /**
* Default read chunk size. This is a conditional default: on 32bit JVMs, it defaults to 100 MB. On 64bit JVMs, it's * Default read chunk size: 2*{@link BufferedIndexInput#MERGE_BUFFER_SIZE}.
* <code>Integer.MAX_VALUE</code>.
*
* @see #setReadChunkSize
*/ */
public static final int DEFAULT_READ_CHUNK_SIZE = Constants.JRE_IS_64BIT ? Integer.MAX_VALUE : 100 * 1024 * 1024; public static final int DEFAULT_READ_CHUNK_SIZE = BufferedIndexInput.MERGE_BUFFER_SIZE * 2;
protected final File directory; // The underlying filesystem directory protected final File directory; // The underlying filesystem directory
protected final Set<String> staleFiles = synchronizedSet(new HashSet<String>()); // Files written, but not yet sync'ed protected final Set<String> staleFiles = synchronizedSet(new HashSet<String>()); // Files written, but not yet sync'ed
@ -370,20 +367,13 @@ public abstract class FSDirectory extends Directory {
* already-opened {@link IndexInput}s. You should call * already-opened {@link IndexInput}s. You should call
* this before attempting to open an index on the * this before attempting to open an index on the
* directory.</p> * directory.</p>
*
* <p> <b>NOTE</b>: This value should be as large as
* possible to reduce any possible performance impact. If
* you still encounter an incorrect OutOfMemoryError,
* trying lowering the chunk size.</p>
*/ */
public final void setReadChunkSize(int chunkSize) { public final void setReadChunkSize(int chunkSize) {
// LUCENE-1566 // LUCENE-1566
if (chunkSize <= 0) { if (chunkSize <= 0) {
throw new IllegalArgumentException("chunkSize must be positive"); throw new IllegalArgumentException("chunkSize must be positive");
} }
if (!Constants.JRE_IS_64BIT) { this.chunkSize = chunkSize;
this.chunkSize = chunkSize;
}
} }
/** /**

6
lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.util;
import java.io.*; import java.io.*;
import java.lang.annotation.*; import java.lang.annotation.*;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.util.*; import java.util.*;
import java.util.concurrent.*; import java.util.concurrent.*;
@ -1135,9 +1136,10 @@ public abstract class LuceneTestCase extends Assert {
FSDirectory d = null; FSDirectory d = null;
try { try {
d = CommandLineUtil.newFSDirectory(clazz, file); d = CommandLineUtil.newFSDirectory(clazz, file);
} catch (Exception e) { } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) {
d = FSDirectory.open(file); Rethrow.rethrow(e);
} }
d.setReadChunkSize(_TestUtil.nextInt(random(), 8, 32678));
return d; return d;
} }