Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-9241: fix tests to pass with -Xmx128m

This commit is contained in:
Robert Muir 2020-03-05 21:29:39 -05:00
parent e5be034df2
commit 9cfdf17b28
No known key found for this signature in database
GPG Key ID: 817AE1DD322D7ECA
22 changed files with 132 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
lucene/analysis/common/src/test/org/apache/lucene/analysis/minhash/MinHashFilterTest.java
View File

@ -99,11 +99,14 @@ public class MinHashFilterTest extends BaseTokenStreamTestCase {
}
assertEquals(100, minSet.size());
assertEquals(0, unadded.size());
}
public void testCollisions() {
HashSet<LongPair> collisionDetection = new HashSet<LongPair>();
unadded = new HashSet<LongPair>();
minSet = new FixedSizeTreeSet<LongPair>(500);
for (int i = 0; i < 1000000; i++) {
HashSet<LongPair> unadded = new HashSet<LongPair>();
FixedSizeTreeSet<LongPair> minSet = new FixedSizeTreeSet<LongPair>(500);
int numElements = TEST_NIGHTLY ? 1000000 : 10000;
for (int i = 0; i < numElements; i++) {
LongPair hash = new LongPair();
MinHashFilter.murmurhash3_x64_128(MinHashFilter.getBytes(i), 0, 4, 0, hash);
collisionDetection.add(hash);
@ -122,9 +125,9 @@ public class MinHashFilterTest extends BaseTokenStreamTestCase {
}
}
}
assertEquals(1000000, collisionDetection.size());
assertEquals(numElements, collisionDetection.size());
assertEquals(500, minSet.size());
assertEquals(999500, unadded.size());
assertEquals(numElements - 500, unadded.size());
LongPair last = null;
LongPair current = null;

26
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.analysis.ja.dict;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.DataInput;
@ -35,7 +36,8 @@ public final class ConnectionCosts {
public static final String HEADER = "kuromoji_cc";
public static final int VERSION = 1;
private final short[][] costs; // array is backward IDs first since get is called using the same backward ID consecutively. maybe doesn't matter.
private final ByteBuffer buffer;
private final int forwardSize;
/**
* @param scheme - scheme for loading resources (FILE or CLASSPATH).
@ -43,24 +45,26 @@ public final class ConnectionCosts {
*/
public ConnectionCosts(BinaryDictionary.ResourceScheme scheme, String path) throws IOException {
InputStream is = null;
short[][] costs = null;
boolean success = false;
try {
is = BinaryDictionary.getResource(scheme, path.replace('.', '/') + FILENAME_SUFFIX);
is = new BufferedInputStream(is);
final DataInput in = new InputStreamDataInput(is);
CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);
int forwardSize = in.readVInt();
forwardSize = in.readVInt();
int backwardSize = in.readVInt();
costs = new short[backwardSize][forwardSize];
int size = forwardSize * backwardSize;
// copy the matrix into a direct byte buffer
final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size*2);
int accum = 0;
for (int j = 0; j < costs.length; j++) {
final short[] a = costs[j];
for (int i = 0; i < a.length; i++) {
for (int j = 0; j < backwardSize; j++) {
for (int i = 0; i < forwardSize; i++) {
accum += in.readZInt();
a[i] = (short)accum;
tmpBuffer.putShort((short) accum);
}
}
buffer = tmpBuffer.asReadOnlyBuffer();
success = true;
} finally {
if (success) {
@ -69,8 +73,6 @@ public final class ConnectionCosts {
IOUtils.closeWhileHandlingException(is);
}
}
this.costs = costs;
}
private ConnectionCosts() throws IOException {
@ -78,7 +80,9 @@ public final class ConnectionCosts {
}
public int get(int forwardId, int backwardId) {
return costs[backwardId][forwardId];
// map 2d matrix into a single dimension short array
int offset = (backwardId * forwardSize + forwardId) * 2;
return buffer.getShort(offset);
}
public static ConnectionCosts getInstance() {

4
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/BinaryDictionaryWriter.java
View File

@ -46,7 +46,7 @@ abstract class BinaryDictionaryWriter {
BinaryDictionaryWriter(Class<? extends BinaryDictionary> implClazz, int size) {
this.implClazz = implClazz;
buffer = ByteBuffer.allocate(size);
buffer = ByteBuffer.allocateDirect(size);
}
/**
@ -97,7 +97,7 @@ abstract class BinaryDictionaryWriter {
// worst case: two short, 3 bytes, and features (all as utf-16)
int worstCase = 4 + 3 + 2*(baseForm.length() + reading.length() + pronunciation.length());
if (worstCase > left) {
ByteBuffer newBuffer = ByteBuffer.allocate(ArrayUtil.oversize(buffer.limit() + worstCase - left, 1));
ByteBuffer newBuffer = ByteBuffer.allocateDirect(ArrayUtil.oversize(buffer.limit() + worstCase - left, 1));
buffer.flip();
newBuffer.put(buffer);
buffer = newBuffer;

21
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.analysis.ja.util;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
@ -31,7 +32,7 @@ import org.apache.lucene.store.OutputStreamDataOutput;
final class ConnectionCostsWriter {
private final short[][] costs; // array is backward IDs first since get is called using the same backward ID consecutively. maybe doesn't matter.
private final ByteBuffer costs; // array is backward IDs first since get is called using the same backward ID consecutively. maybe doesn't matter.
private final int forwardSize;
private final int backwardSize;
/**
@ -40,11 +41,12 @@ final class ConnectionCostsWriter {
ConnectionCostsWriter(int forwardSize, int backwardSize) {
this.forwardSize = forwardSize;
this.backwardSize = backwardSize;
this.costs = new short[backwardSize][forwardSize];
this.costs = ByteBuffer.allocateDirect(2 * backwardSize * forwardSize);
}
public void add(int forwardId, int backwardId, int cost) {
this.costs[backwardId][forwardId] = (short)cost;
int offset = (backwardId * forwardSize + forwardId) * 2;
costs.putShort(offset, (short) cost);
}
public void write(Path baseDir) throws IOException {
@ -57,14 +59,11 @@ final class ConnectionCostsWriter {
out.writeVInt(forwardSize);
out.writeVInt(backwardSize);
int last = 0;
assert costs.length == backwardSize;
for (short[] a : costs) {
assert a.length == forwardSize;
for (short cost : a) {
int delta = (int) cost - last;
out.writeZInt(delta);
last = cost;
}
for (int i = 0; i < costs.limit() / 2; i++) {
short cost = costs.getShort(i * 2);
int delta = (int) cost - last;
out.writeZInt(delta);
last = cost;
}
}
}

14
lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
View File

@ -48,17 +48,9 @@ public class
TestJapaneseTokenizer extends BaseTokenStreamTestCase {
public static UserDictionary readDict() {
InputStream is = TestJapaneseTokenizer.class.getResourceAsStream("userdict.txt");
if (is == null) {
throw new RuntimeException("Cannot find userdict.txt in test classpath!");
}
try {
try {
Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
return UserDictionary.open(reader);
} finally {
is.close();
}
try (InputStream stream = TestJapaneseTokenizer.class.getResourceAsStream("userdict.txt");
Reader reader = new InputStreamReader(stream, StandardCharsets.UTF_8)) {
return UserDictionary.open(reader);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}

2
lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java
View File

@ -44,7 +44,6 @@ public final class ConnectionCosts {
*/
public ConnectionCosts(BinaryDictionary.ResourceScheme scheme, String resourcePath) throws IOException {
InputStream is = null;
ByteBuffer buffer;
boolean success = false;
try {
is = BinaryDictionary.getResource(scheme, resourcePath.replace('.', '/') + FILENAME_SUFFIX);
@ -73,7 +72,6 @@ public final class ConnectionCosts {
IOUtils.closeWhileHandlingException(is);
}
}
this.buffer = buffer;
}
private ConnectionCosts() throws IOException {

4
lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java
View File

@ -49,7 +49,7 @@ abstract class BinaryDictionaryWriter {
BinaryDictionaryWriter(Class<? extends BinaryDictionary> implClazz, int size) {
this.implClazz = implClazz;
buffer = ByteBuffer.allocate(size);
buffer = ByteBuffer.allocateDirect(size);
}
/**
@ -96,7 +96,7 @@ abstract class BinaryDictionaryWriter {
// worst case, 3 short + 4 bytes and features (all as utf-16)
int worstCase = 9 + 2*(expression.length() + reading.length());
if (worstCase > left) {
ByteBuffer newBuffer = ByteBuffer.allocate(ArrayUtil.oversize(buffer.limit() + worstCase - left, 1));
ByteBuffer newBuffer = ByteBuffer.allocateDirect(ArrayUtil.oversize(buffer.limit() + worstCase - left, 1));
buffer.flip();
newBuffer.put(buffer);
buffer = newBuffer;

21
lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsWriter.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.analysis.ko.util;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
@ -30,7 +31,7 @@ import org.apache.lucene.store.OutputStreamDataOutput;
final class ConnectionCostsWriter {
private final short[][] costs; // array is backward IDs first since get is called using the same backward ID consecutively. maybe doesn't matter.
private final ByteBuffer costs; // array is backward IDs first since get is called using the same backward ID consecutively. maybe doesn't matter.
private final int forwardSize;
private final int backwardSize;
/**
@ -39,11 +40,12 @@ final class ConnectionCostsWriter {
ConnectionCostsWriter(int forwardSize, int backwardSize) {
this.forwardSize = forwardSize;
this.backwardSize = backwardSize;
this.costs = new short[backwardSize][forwardSize];
this.costs = ByteBuffer.allocateDirect(2 * backwardSize * forwardSize);
}
public void add(int forwardId, int backwardId, int cost) {
this.costs[backwardId][forwardId] = (short)cost;
int offset = (backwardId * forwardSize + forwardId) * 2;
costs.putShort(offset, (short) cost);
}
public void write(Path baseDir) throws IOException {
@ -56,14 +58,11 @@ final class ConnectionCostsWriter {
out.writeVInt(forwardSize);
out.writeVInt(backwardSize);
int last = 0;
assert costs.length == backwardSize;
for (short[] a : costs) {
assert a.length == forwardSize;
for (short cost : a) {
int delta = (int) cost - last;
out.writeZInt(delta);
last = cost;
}
for (int i = 0; i < costs.limit() / 2; i++) {
short cost = costs.getShort(i * 2);
int delta = (int) cost - last;
out.writeZInt(delta);
last = cost;
}
}
}

2
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
View File

@ -61,7 +61,7 @@ import org.apache.lucene.util.TestUtil;
/**
* Test very simply that perf tasks - simple algorithms - are doing what they should.
*/
@LuceneTestCase.SuppressCodecs("SimpleText")
@LuceneTestCase.SuppressCodecs({"SimpleText", "Direct"})
public class TestPerfTasksLogic extends BenchmarkTestCase {
@Override

2
lucene/core/src/java/org/apache/lucene/util/automaton/ByteRunAutomaton.java
View File

@ -41,6 +41,6 @@ public class ByteRunAutomaton extends RunAutomaton {
p = step(p, s[i] & 0xFF);
if (p == -1) return false;
}
return accept[p];
return accept.get(p);
}
}

4
lucene/core/src/java/org/apache/lucene/util/automaton/CharacterRunAutomaton.java
View File

@ -49,7 +49,7 @@ public class CharacterRunAutomaton extends RunAutomaton {
p = step(p, cp = s.codePointAt(i));
if (p == -1) return false;
}
return accept[p];
return accept.get(p);
}
/**
@ -62,6 +62,6 @@ public class CharacterRunAutomaton extends RunAutomaton {
p = step(p, cp = Character.codePointAt(s, i, l));
if (p == -1) return false;
}
return accept[p];
return accept.get(p);
}
}

17
lucene/core/src/java/org/apache/lucene/util/automaton/RunAutomaton.java
View File

@ -32,6 +32,7 @@ package org.apache.lucene.util.automaton;
import java.util.Arrays;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.RamUsageEstimator;
/**
@ -45,7 +46,7 @@ public abstract class RunAutomaton implements Accountable {
final Automaton automaton;
final int alphabetSize;
final int size;
final boolean[] accept;
final FixedBitSet accept;
final int[] transitions; // delta(state,c) = transitions[state*points.length +
// getCharClass(c)]
final int[] points; // char interval start points
@ -75,12 +76,14 @@ public abstract class RunAutomaton implements Accountable {
this.automaton = a;
points = a.getStartPoints();
size = Math.max(1,a.getNumStates());
accept = new boolean[size];
accept = new FixedBitSet(size);
transitions = new int[size * points.length];
Arrays.fill(transitions, -1);
Transition transition = new Transition();
for (int n=0;n<size;n++) {
accept[n] = a.isAccept(n);
if (a.isAccept(n)) {
accept.set(n);
}
transition.source = n;
transition.transitionUpto = -1;
for (int c = 0; c < points.length; c++) {
@ -112,7 +115,7 @@ public abstract class RunAutomaton implements Accountable {
b.append("initial state: 0\n");
for (int i = 0; i < size; i++) {
b.append("state ").append(i);
if (accept[i]) b.append(" [accept]:\n");
if (accept.get(i)) b.append(" [accept]:\n");
else b.append(" [reject]:\n");
for (int j = 0; j < points.length; j++) {
int k = transitions[i * points.length + j];
@ -145,7 +148,7 @@ public abstract class RunAutomaton implements Accountable {
* Returns acceptance status for given state.
*/
public final boolean isAccept(int state) {
return accept[state];
return accept.get(state);
}
/**
@ -208,7 +211,7 @@ public abstract class RunAutomaton implements Accountable {
if (alphabetSize != other.alphabetSize) return false;
if (size != other.size) return false;
if (!Arrays.equals(points, other.points)) return false;
if (!Arrays.equals(accept, other.accept)) return false;
if (!accept.equals(other.accept)) return false;
if (!Arrays.equals(transitions, other.transitions)) return false;
return true;
}
@ -216,7 +219,7 @@ public abstract class RunAutomaton implements Accountable {
@Override
public long ramBytesUsed() {
return BASE_RAM_BYTES +
RamUsageEstimator.sizeOfObject(accept) +
accept.ramBytesUsed() +
RamUsageEstimator.sizeOfObject(automaton) +
RamUsageEstimator.sizeOfObject(classmap) +
RamUsageEstimator.sizeOfObject(points) +

46
lucene/core/src/test/org/apache/lucene/codecs/lucene80/TestIndexedDISI.java
View File

@ -17,8 +17,6 @@
package org.apache.lucene.codecs.lucene80;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import org.apache.lucene.search.DocIdSetIterator;
@ -27,9 +25,11 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.SparseFixedBitSet;
import org.apache.lucene.util.TestUtil;
// Copied from the lucene70 package for separation of codec-code
@ -37,7 +37,7 @@ public class TestIndexedDISI extends LuceneTestCase {
public void testEmpty() throws IOException {
int maxDoc = TestUtil.nextInt(random(), 1, 100000);
FixedBitSet set = new FixedBitSet(maxDoc);
BitSet set = new SparseFixedBitSet(maxDoc);
try (Directory dir = newDirectory()) {
doTest(set, dir);
}
@ -48,7 +48,7 @@ public class TestIndexedDISI extends LuceneTestCase {
public void testEmptyBlocks() throws IOException {
final int B = 65536;
int maxDoc = B*11;
FixedBitSet set = new FixedBitSet(maxDoc);
BitSet set = new SparseFixedBitSet(maxDoc);
// block 0: EMPTY
set.set(B+5); // block 1: SPARSE
// block 2: EMPTY
@ -83,7 +83,7 @@ public class TestIndexedDISI extends LuceneTestCase {
public void testLastEmptyBlocks() throws IOException {
final int B = 65536;
int maxDoc = B*3;
FixedBitSet set = new FixedBitSet(maxDoc);
BitSet set = new SparseFixedBitSet(maxDoc);
for (int docID = 0 ; docID < B*2 ; docID++) { // first 2 blocks are ALL
set.set(docID);
}
@ -96,7 +96,7 @@ public class TestIndexedDISI extends LuceneTestCase {
}
// Checks that advance after the end of the blocks has been reached has the correct behaviour
private void assertAdvanceBeyondEnd(FixedBitSet set, Directory dir) throws IOException {
private void assertAdvanceBeyondEnd(BitSet set, Directory dir) throws IOException {
final int cardinality = set.cardinality();
final byte denseRankPower = 9; // Not tested here so fixed to isolate factors
long length;
@ -127,7 +127,7 @@ public class TestIndexedDISI extends LuceneTestCase {
@Nightly
public void testRandomBlocks() throws IOException {
final int BLOCKS = 5;
FixedBitSet set = createSetWithRandomBlocks(BLOCKS);
BitSet set = createSetWithRandomBlocks(BLOCKS);
try (Directory dir = newDirectory()) {
doTestAllSingleJump(set, dir);
}
@ -138,7 +138,7 @@ public class TestIndexedDISI extends LuceneTestCase {
final int BLOCKS = 10;
final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7)+7); // sane + chance of disable
FixedBitSet set = createSetWithRandomBlocks(BLOCKS);
BitSet set = createSetWithRandomBlocks(BLOCKS);
try (Directory dir = newDirectory()) {
final int cardinality = set.cardinality();
int jumpTableEntryCount;
@ -158,9 +158,9 @@ public class TestIndexedDISI extends LuceneTestCase {
}
}
private FixedBitSet createSetWithRandomBlocks(int blockCount) {
private BitSet createSetWithRandomBlocks(int blockCount) {
final int B = 65536;
FixedBitSet set = new FixedBitSet(blockCount * B);
BitSet set = new SparseFixedBitSet(blockCount * B);
for (int block = 0; block < blockCount; block++) {
switch (random().nextInt(4)) {
case 0: { // EMPTY
@ -191,7 +191,7 @@ public class TestIndexedDISI extends LuceneTestCase {
}
private void doTestAllSingleJump(FixedBitSet set, Directory dir) throws IOException {
private void doTestAllSingleJump(BitSet set, Directory dir) throws IOException {
final int cardinality = set.cardinality();
final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7)+7); // sane + chance of disable
long length;
@ -222,7 +222,7 @@ public class TestIndexedDISI extends LuceneTestCase {
public void testOneDoc() throws IOException {
int maxDoc = TestUtil.nextInt(random(), 1, 100000);
FixedBitSet set = new FixedBitSet(maxDoc);
BitSet set = new SparseFixedBitSet(maxDoc);
set.set(random().nextInt(maxDoc));
try (Directory dir = newDirectory()) {
doTest(set, dir);
@ -231,7 +231,7 @@ public class TestIndexedDISI extends LuceneTestCase {
public void testTwoDocs() throws IOException {
int maxDoc = TestUtil.nextInt(random(), 1, 100000);
FixedBitSet set = new FixedBitSet(maxDoc);
BitSet set = new SparseFixedBitSet(maxDoc);
set.set(random().nextInt(maxDoc));
set.set(random().nextInt(maxDoc));
try (Directory dir = newDirectory()) {
@ -250,7 +250,7 @@ public class TestIndexedDISI extends LuceneTestCase {
public void testHalfFull() throws IOException {
int maxDoc = TestUtil.nextInt(random(), 1, 100000);
FixedBitSet set = new FixedBitSet(maxDoc);
BitSet set = new SparseFixedBitSet(maxDoc);
for (int i = random().nextInt(2); i < maxDoc; i += TestUtil.nextInt(random(), 1, 3)) {
set.set(i);
}
@ -371,7 +371,7 @@ public class TestIndexedDISI extends LuceneTestCase {
}
private void createAndOpenDISI(byte denseRankPowerWrite, byte denseRankPowerRead) throws IOException {
FixedBitSet set = new FixedBitSet(10);
BitSet set = new FixedBitSet(10);
set.set(set.length()-1);
try (Directory dir = newDirectory()) {
long length;
@ -423,24 +423,22 @@ public class TestIndexedDISI extends LuceneTestCase {
private void doTestRandom(Directory dir) throws IOException {
Random random = random();
List<Integer> docs = new ArrayList<>();
final int maxStep = TestUtil.nextInt(random, 1, 1 << TestUtil.nextInt(random, 2, 20));
final int numDocs = TestUtil.nextInt(random, 1, Math.min(100000, Integer.MAX_VALUE / maxStep));
BitSet docs = new SparseFixedBitSet(numDocs * (maxStep + 1));
int lastDoc = -1;
for (int doc = -1, i = 0; i < numDocs; ++i) {
doc += TestUtil.nextInt(random, 1, maxStep);
docs.add(doc);
}
final int maxDoc = docs.get(docs.size() - 1) + TestUtil.nextInt(random, 1, 100);
FixedBitSet set = new FixedBitSet(maxDoc);
for (int doc : docs) {
set.set(doc);
docs.set(doc);
lastDoc = doc;
}
final int maxDoc = lastDoc + TestUtil.nextInt(random, 1, 100);
BitSet set = BitSet.of(new BitSetIterator(docs, docs.approximateCardinality()), maxDoc);
doTest(set, dir);
}
private void doTest(FixedBitSet set, Directory dir) throws IOException {
private void doTest(BitSet set, Directory dir) throws IOException {
final int cardinality = set.cardinality();
final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7)+7); // sane + chance of disable
long length;

3
lucene/core/src/test/org/apache/lucene/search/TestAutomatonQuery.java
View File

@ -241,8 +241,9 @@ public class TestAutomatonQuery extends LuceneTestCase {
}
public void testBiggishAutomaton() {
int numTerms = TEST_NIGHTLY ? 3000 : 500;
List<BytesRef> terms = new ArrayList<>();
while (terms.size() < 3000) {
while (terms.size() < numTerms) {
terms.add(new BytesRef(TestUtil.randomUnicodeString(random())));
}
Collections.sort(terms);

2
lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java
View File

@ -67,7 +67,7 @@ public final class TestByteBuffersDataInput extends RandomizedTest {
ByteBuffersDataOutput dst = new ByteBuffersDataOutput();
long seed = randomLong();
int max = 1_000_000;
int max = LuceneTestCase.TEST_NIGHTLY ? 1_000_000 : 100_000;
List<IOConsumer<DataInput>> reply =
TestByteBuffersDataOutput.addRandomData(dst, new Xoroshiro128PlusRandom(seed), max);

8
lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataOutput.java
View File

@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Assert;
import org.junit.Test;
@ -122,7 +123,12 @@ public final class TestByteBuffersDataOutput extends BaseDataOutputTestCase<Byte
public void testLargeArrayAdd() {
ByteBuffersDataOutput o = new ByteBuffersDataOutput();
int MB = 1024 * 1024;
byte [] bytes = randomBytesOfLength(5 * MB, 15 * MB);
final byte [] bytes;
if (LuceneTestCase.TEST_NIGHTLY) {
bytes = randomBytesOfLength(5 * MB, 15 * MB);
} else {
bytes = randomBytesOfLength(MB/2, MB);
}
int offset = randomIntBetween(0, 100);
int len = bytes.length - offset;
o.writeBytes(bytes, offset, len);

9
lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
View File

@ -81,16 +81,17 @@ public class TestOfflineSorter extends LuceneTestCase {
if (random().nextBoolean()) {
return null;
} else {
return new ThreadPoolExecutor(1, TestUtil.nextInt(random(), 2, 6), Long.MAX_VALUE, TimeUnit.MILLISECONDS,
int maxThreads = TEST_NIGHTLY ? TestUtil.nextInt(random(), 2, 6) : 2;
return new ThreadPoolExecutor(1, maxThreads, Long.MAX_VALUE, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamedThreadFactory("TestIndexSearcher"));
new NamedThreadFactory("TestOfflineSorter"));
}
}
@Slow
public void testIntermediateMerges() throws Exception {
// Sort 20 mb worth of data with 1mb buffer, binary merging.
try (Directory dir = newDirectory()) {
try (Directory dir = newFSDirectory(createTempDir())) {
ExecutorService exec = randomExecutorServiceOrNull();
SortInfo info = checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 2, -1, exec, TestUtil.nextInt(random(), 1, 4)),
generateRandom((int)OfflineSorter.MB * 20));
@ -104,7 +105,7 @@ public class TestOfflineSorter extends LuceneTestCase {
@Slow
public void testSmallRandom() throws Exception {
// Sort 20 mb worth of data with 1mb buffer.
try (Directory dir = newDirectory()) {
try (Directory dir = newFSDirectory(createTempDir())) {
ExecutorService exec = randomExecutorServiceOrNull();
SortInfo sortInfo = checkSort(dir, new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.MAX_TEMPFILES, -1, exec, TestUtil.nextInt(random(), 1, 4)),
generateRandom((int)OfflineSorter.MB * 20));

14
lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java
View File

@ -45,7 +45,12 @@ public class TestPagedBytes extends LuceneTestCase {
final int blockSize = 1 << blockBits;
final PagedBytes p = new PagedBytes(blockBits);
final IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
final int numBytes = TestUtil.nextInt(random, 2, 10000000);
final int numBytes;
if (TEST_NIGHTLY) {
numBytes = TestUtil.nextInt(random(), 2, 10_000_000);
} else {
numBytes = TestUtil.nextInt(random(), 2, 1_000_000);
}
final byte[] answer = new byte[numBytes];
random.nextBytes(answer);
@ -105,7 +110,12 @@ public class TestPagedBytes extends LuceneTestCase {
final int blockSize = 1 << blockBits;
final PagedBytes p = new PagedBytes(blockBits);
final DataOutput out = p.getDataOutput();
final int numBytes = random().nextInt(10000000);
final int numBytes;
if (TEST_NIGHTLY) {
numBytes = random().nextInt(10_000_000);
} else {
numBytes = random().nextInt(1_000_000);
}
final byte[] answer = new byte[numBytes];
random().nextBytes(answer);

11
lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java
View File

@ -55,10 +55,13 @@ public class TestDeterminizeLexicon extends LuceneTestCase {
for (String s : terms) {
assertTrue(Operations.run(lex, s));
}
final ByteRunAutomaton lexByte = new ByteRunAutomaton(lex, false, 1000000);
for (String s : terms) {
byte bytes[] = s.getBytes(StandardCharsets.UTF_8);
assertTrue(lexByte.run(bytes, 0, bytes.length));
if (TEST_NIGHTLY) {
// TODO: very wasteful of RAM to do this without minimizing first.
final ByteRunAutomaton lexByte = new ByteRunAutomaton(lex, false, 1000000);
for (String s : terms) {
byte bytes[] = s.getBytes(StandardCharsets.UTF_8);
assertTrue(lexByte.run(bytes, 0, bytes.length));
}
}
}
}

7
lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCharBlockArray.java
View File

@ -33,7 +33,12 @@ public class TestCharBlockArray extends FacetTestCase {
CharBlockArray array = new CharBlockArray();
StringBuilder builder = new StringBuilder();
final int n = 100 * 1000;
final int n;
if (TEST_NIGHTLY) {
n = 100 * 1000;
} else {
n = 1000;
}
byte[] buffer = new byte[50];

6
lucene/monitor/src/test/org/apache/lucene/monitor/TestConcurrentQueryLoader.java
View File

@ -20,13 +20,15 @@ package org.apache.lucene.monitor;
public class TestConcurrentQueryLoader extends MonitorTestBase {
public void testLoading() throws Exception {
// TODO: there are problems if this thing hits OOM (it hides it)
int numQueries = TEST_NIGHTLY ? 2000 : 200;
try (Monitor monitor = newMonitor()) {
try (ConcurrentQueryLoader loader = new ConcurrentQueryLoader(monitor)) {
for (int i = 0; i < 2000; i++) {
for (int i = 0; i < numQueries; i++) {
loader.add(new MonitorQuery(Integer.toString(i), parse("\"test " + i + "\"")));
}
}
assertEquals(2000, monitor.getQueryCount());
assertEquals(numQueries, monitor.getQueryCount());
}
}

2
lucene/monitor/src/test/org/apache/lucene/monitor/TestMonitor.java
View File

@ -130,6 +130,8 @@ public class TestMonitor extends MonitorTestBase {
}
// takes huge amounts of ram. TODO: what is this test doing?
@Nightly
public void testUpdateReporting() throws IOException {
List<MonitorQuery> queries = new ArrayList<>(10400);