diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayLikeObjectTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayLikeObjectTestCase.java
index 1be3c024710..69b40f141e2 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayLikeObjectTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayLikeObjectTestCase.java
@@ -39,9 +39,9 @@ public abstract class ArrayLikeObjectTestCase extends ScriptTestCase {
      */
     protected abstract String valueCtorCall(String valueType, int size);
     /**
-     * The type of the exception thrown by out of bounds accesses;
+     * Matcher for the message of the out of bounds exceptions thrown for too negative or too positive offsets.
      */
-    protected abstract Matcher<? super IndexOutOfBoundsException> outOfBoundsExceptionMatcher(int index, int size);
+    protected abstract Matcher<String> outOfBoundsExceptionMessageMatcher(int index, int size);
 
     private void arrayLoadStoreTestCase(boolean declareAsDef, String valueType, Object val, @Nullable Number valPlusOne) {
         String declType = declareAsDef ? "def" : declType(valueType);
@@ -80,7 +80,7 @@ public abstract class ArrayLikeObjectTestCase extends ScriptTestCase {
         IndexOutOfBoundsException e = expectScriptThrows(IndexOutOfBoundsException.class,
                 () -> exec(script, singletonMap("val", val), true));
         try {
-            assertThat(e, outOfBoundsExceptionMatcher(index, 5));
+            assertThat(e.getMessage(), outOfBoundsExceptionMessageMatcher(index, 5));
         } catch (AssertionError ae) {
             ae.addSuppressed(e);   // Mark the exception we are testing as suppressed so we get its stack trace. If it has one :(
             throw ae;
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
index 350f27f44fc..fe2ee1683bb 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
@@ -24,9 +24,7 @@ import org.hamcrest.Matcher;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodType;
 
-import static org.hamcrest.Matchers.both;
-import static org.hamcrest.Matchers.hasToString;
-import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.equalTo;
 
 /** Tests for working with arrays. */
 public class ArrayTests extends ArrayLikeObjectTestCase {
@@ -41,9 +39,8 @@ public class ArrayTests extends ArrayLikeObjectTestCase {
     }
 
     @Override
-    protected Matcher<? super IndexOutOfBoundsException> outOfBoundsExceptionMatcher(int index, int size) {
-        return both(instanceOf(ArrayIndexOutOfBoundsException.class))
-                .and(hasToString("java.lang.ArrayIndexOutOfBoundsException: " + index));
+    protected Matcher<String> outOfBoundsExceptionMessageMatcher(int index, int size) {
+        return equalTo(Integer.toString(index));
     }
 
     public void testArrayLengthHelper() throws Throwable {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ListTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ListTests.java
index 7b57e1d0d0a..79112d8f58e 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ListTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ListTests.java
@@ -23,10 +23,9 @@ import org.hamcrest.Matcher;
 
 import java.util.Arrays;
 
-import static org.hamcrest.Matchers.both;
-import static org.hamcrest.Matchers.either;
-import static org.hamcrest.Matchers.hasToString;
-import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 /** Tests for working with lists. */
 public class ListTests extends ArrayLikeObjectTestCase {
@@ -55,15 +54,20 @@ public class ListTests extends ArrayLikeObjectTestCase {
     }
 
     @Override
-    protected Matcher<? super IndexOutOfBoundsException> outOfBoundsExceptionMatcher(int index, int size) {
-        if (index > size) {
-            return hasToString("java.lang.IndexOutOfBoundsException: Index: " + index + ", Size: " + size);
+    protected Matcher<String> outOfBoundsExceptionMessageMatcher(int index, int size) {
+        if ("1.8".equals(Runtime.class.getPackage().getSpecificationVersion())) {
+            // 1.8 and below aren't as clean as 1.9+
+            if (index > size) {
+                return equalTo("Index: " + index + ", Size: " + size);
+            } else {
+                Matcher<String> m = equalTo(Integer.toString(index));
+                // If we set -XX:-OmitStackTraceInFastThrow we wouldn't need this
+                m = anyOf(m, nullValue());
+                return m;
+            }
         } else {
-            Matcher<? super IndexOutOfBoundsException> m = both(instanceOf(ArrayIndexOutOfBoundsException.class))
-                    .and(hasToString("java.lang.ArrayIndexOutOfBoundsException: " + index));
-            // If we set -XX:-OmitStackTraceInFastThrow we wouldn't need this
-            m = either(m).or(instanceOf(ArrayIndexOutOfBoundsException.class));
-            return m;
+            // Starting with 1.9 it gets nicer
+            return equalTo("Index " + index + " out-of-bounds for length " + size);
         }
     }