LUCENE-5844: ArrayUtil.grow/oversize now returns at most Integer.MAX_VALUE - 8

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1612935 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -205,6 +205,10 @@ Bug Fixes
 
 * LUCENE-5838: Fix hunspell when the .aff file has over 64k affixes. (Robert Muir)
 
+* LUCENE-5844: ArrayUtil.grow/oversize now returns a maximum of
+  Integer.MAX_VALUE - 8 for the maximum array size.  (Robert Muir,
+  Mike McCandless)
+
 Test Framework
 
 * LUCENE-5786: Unflushed/ truncated events file (hung testing subprocess).

lucene/core/src/java/org/apache/lucene/util/ArrayUtil.java

@@ -28,13 +28,10 @@ import java.util.Comparator;
 
 public final class ArrayUtil {
 
-  /** Maximum length for an array; we set this to "a
-   *  bit" below Integer.MAX_VALUE because the exact max
-   *  allowed byte[] is JVM dependent, so we want to avoid
-   *  a case where a large value worked during indexing on
-   *  one JVM but failed later at search time with a
-   *  different JVM. */
-  public static final int MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 256;
+  /** Maximum length for an array (Integer.MAX_VALUE - 8).  stackoverflow
+   *  consensus seems to be this value and it's also what ArrayList.java
+   *  uses as its limit.  */
+  public static final int MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 8;
 
   private ArrayUtil() {} // no instance
 
@@ -169,6 +166,10 @@ public final class ArrayUtil {
       return 0;
     }
 
+    if (minTargetSize > MAX_ARRAY_LENGTH) {
+      throw new IllegalArgumentException("requested array size " + minTargetSize + " exceeds maximum array in java (" + MAX_ARRAY_LENGTH + ")");
+    }
+
     // asymptotic exponential growth by 1/8th, favors
     // spending a bit more CPU to not tie up too much wasted
     // RAM:
@@ -184,9 +185,9 @@ public final class ArrayUtil {
     int newSize = minTargetSize + extra;
 
     // add 7 to allow for worst case byte alignment addition below:
-    if (newSize+7 < 0) {
-      // int overflowed -- return max allowed array size
-      return Integer.MAX_VALUE;
+    if (newSize+7 < 0 || newSize+7 > MAX_ARRAY_LENGTH) {
+      // int overflowed, or we exceeded the maximum array length
+      return MAX_ARRAY_LENGTH;
     }
 
     if (Constants.JRE_IS_64BIT) {

lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java

@@ -43,21 +43,13 @@ public abstract class PriorityQueue<T> {
       // We allocate 1 extra to avoid if statement in top()
       heapSize = 2;
     } else {
-      if (maxSize > ArrayUtil.MAX_ARRAY_LENGTH) {
-        // Don't wrap heapSize to -1, in this case, which
-        // causes a confusing NegativeArraySizeException.
-        // Note that very likely this will simply then hit
-        // an OOME, but at least that's more indicative to
-        // caller that this values is too big.  We don't +1
-        // in this case, but it's very unlikely in practice
-        // one will actually insert this many objects into
-        // the PQ:
+      // NOTE: we add +1 because all access to heap is
+      // 1-based not 0-based.  heap[0] is unused.
+      heapSize = maxSize + 1;
+
+      if (heapSize > ArrayUtil.MAX_ARRAY_LENGTH) {
         // Throw exception to prevent confusing OOME:
-        throw new IllegalArgumentException("maxSize must be <= " + ArrayUtil.MAX_ARRAY_LENGTH + "; got: " + maxSize);
-      } else {
-        // NOTE: we add +1 because all access to heap is
-        // 1-based not 0-based.  heap[0] is unused.
-        heapSize = maxSize + 1;
+        throw new IllegalArgumentException("maxSize must be <= " + (ArrayUtil.MAX_ARRAY_LENGTH-1) + "; got: " + maxSize);
       }
     }
     // T is unbounded type, so this unchecked cast works always:

lucene/core/src/test/org/apache/lucene/util/TestArrayUtil.java

@@ -29,7 +29,7 @@ public class TestArrayUtil extends LuceneTestCase {
     long copyCost = 0;
 
     // Make sure ArrayUtil hits Integer.MAX_VALUE, if we insist:
-    while(currentSize != Integer.MAX_VALUE) {
+    while (currentSize != ArrayUtil.MAX_ARRAY_LENGTH) {
       int nextSize = ArrayUtil.oversize(1+currentSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
       assertTrue(nextSize > currentSize);
       if (currentSize > 0) {
@@ -44,11 +44,24 @@ public class TestArrayUtil extends LuceneTestCase {
   public void testMaxSize() {
     // intentionally pass invalid elemSizes:
     for(int elemSize=0;elemSize<10;elemSize++) {
-      assertEquals(Integer.MAX_VALUE, ArrayUtil.oversize(Integer.MAX_VALUE, elemSize));
-      assertEquals(Integer.MAX_VALUE, ArrayUtil.oversize(Integer.MAX_VALUE-1, elemSize));
+      assertEquals(ArrayUtil.MAX_ARRAY_LENGTH, ArrayUtil.oversize(ArrayUtil.MAX_ARRAY_LENGTH, elemSize));
+      assertEquals(ArrayUtil.MAX_ARRAY_LENGTH, ArrayUtil.oversize(ArrayUtil.MAX_ARRAY_LENGTH-1, elemSize));
     }
   }
 
+  public void testTooBig() {
+    try {
+      ArrayUtil.oversize(ArrayUtil.MAX_ARRAY_LENGTH+1, 1);
+      fail("did not hit exception");
+    } catch (IllegalArgumentException iae) {
+      // expected
+    }
+  }
+
+  public void testExactLimit() {
+    assertEquals(ArrayUtil.MAX_ARRAY_LENGTH, ArrayUtil.oversize(ArrayUtil.MAX_ARRAY_LENGTH, 1));
+  }
+
   public void testInvalidElementSizes() {
     final Random rnd = random();
     final int num = atLeast(10000);