diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java
index 4a59db64640..e021df52c60 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/LoggingListener.java
@@ -127,8 +127,10 @@ public class LoggingListener extends RunListener {
         final String[] loggersAndLevels = testLogging.value().split(",");
         for (final String loggerAndLevel : loggersAndLevels) {
             final String[] loggerAndLevelArray = loggerAndLevel.split(":");
-            if (loggerAndLevelArray.length >= 2) {
+            if (loggerAndLevelArray.length == 2) {
                 map.put(loggerAndLevelArray[0], loggerAndLevelArray[1]);
+            } else {
+                throw new IllegalArgumentException("invalid test logging annotation [" + loggerAndLevel + "]");
             }
         }
         return map;
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java
index b2fa68359b1..0845fc2546f 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/LoggingListenerTests.java
@@ -163,8 +163,33 @@ public class LoggingListenerTests extends ESTestCase {
         assertThat(abcLogger.getLevel(), equalTo(level));
     }
 
+    public void testInvalidClassTestLoggingAnnotation() throws Exception {
+        final LoggingListener loggingListener = new LoggingListener();
+
+        final Description suiteDescription = Description.createSuiteDescription(InvalidClass.class);
+
+        final IllegalArgumentException e =
+            expectThrows(IllegalArgumentException.class, () -> loggingListener.testRunStarted(suiteDescription));
+        assertThat(e.getMessage(), equalTo("invalid test logging annotation [abc]"));
+    }
+
+    public void testInvalidMethodTestLoggingAnnotation() throws Exception {
+        final LoggingListener loggingListener = new LoggingListener();
+
+        final Description suiteDescription = Description.createSuiteDescription(InvalidMethod.class);
+
+        loggingListener.testRunStarted(suiteDescription);
+
+        final Method method = InvalidMethod.class.getMethod("invalidMethod");
+        final TestLogging annotation = method.getAnnotation(TestLogging.class);
+        Description testDescription = Description.createTestDescription(InvalidMethod.class, "invalidMethod", annotation);
+        final IllegalArgumentException e =
+            expectThrows(IllegalArgumentException.class, () -> loggingListener.testStarted(testDescription));
+        assertThat(e.getMessage(), equalTo("invalid test logging annotation [abc:INFO:WARN]"));
+    }
+
     /**
-     * dummy class used to create a junit suite description that has the @TestLogging annotation
+     * Dummy class used to create a JUnit suite description that has the {@link TestLogging} annotation.
      */
     @TestLogging("abc:WARN,foo:WARN,foo.bar:ERROR")
     public static class AnnotatedTestClass {
@@ -172,17 +197,43 @@ public class LoggingListenerTests extends ESTestCase {
     }
 
     /**
-     * dummy class used to create a junit suite description that doesn't have the @TestLogging annotation, but its test methods have it
+     * Dummy class used to create a JUnit suite description that doesn't have the {@link TestLogging} annotation, but its test methods have
+     * it.
      */
     public static class TestClass {
 
         @SuppressWarnings("unused")
         @TestLogging("xyz:TRACE,foo:WARN,foo.bar:ERROR")
-        public void annotatedTestMethod() {}
+        public void annotatedTestMethod() {
+
+        }
 
         @SuppressWarnings("unused")
         @TestLogging("abc:TRACE,xyz:DEBUG")
-        public void annotatedTestMethod2() {}
+        public void annotatedTestMethod2() {
+
+        }
+
+    }
+
+    /**
+     * Dummy class with an invalid {@link TestLogging} annotation.
+     */
+    @TestLogging("abc")
+    public static class InvalidClass {
+
+    }
+
+    /**
+     * Dummy class with an invalid {@link TestLogging} annotation on a method.
+     */
+    public static class InvalidMethod {
+
+        @SuppressWarnings("unused")
+        @TestLogging("abc:INFO:WARN")
+        public void invalidMethod() {
+
+        }
 
     }