diff --git a/pom.xml b/pom.xml
index a83e7988622..a5d3b0a433c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,89 +101,6 @@
             <plugin>
                 <groupId>com.carrotsearch.randomizedtesting</groupId>
                 <artifactId>junit4-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>tests</id>
-                        <phase>test</phase>
-                        <goals>
-                            <goal>junit4</goal>
-                        </goals>
-                        <configuration>
-                            <heartbeat>20</heartbeat>
-                            <jvmOutputAction>pipe,warn</jvmOutputAction>
-                            <leaveTemporary>true</leaveTemporary>
-                            <listeners>
-                                <report-ant-xml mavenExtensions="true"
-                                                dir="${project.build.directory}/surefire-reports"/>
-                                <report-text
-                                        showThrowable="true"
-                                        showStackTraces="true"
-                                        showOutput="${tests.output}"
-                                        showStatusOk="false"
-                                        showStatusError="true"
-                                        showStatusFailure="true"
-                                        showStatusIgnored="true"
-                                        showSuiteSummary="true"
-                                        timestamps="false"/>
-                                <report-execution-times file="${basedir}/.local-execution-hints.log"/>
-                            </listeners>
-                            <assertions>
-                                <enable/>
-                            </assertions>
-                            <parallelism>1</parallelism>
-                            <balancers>
-                                <execution-times>
-                                    <fileset dir="${basedir}" includes=".local-execution-hints.log"/>
-                                </execution-times>
-                            </balancers>
-                            <includes>
-                                <include>**/*Tests.class</include>
-                                <include>**/*Test.class</include>
-                            </includes>
-                            <excludes>
-                                <exclude>**/Abstract*.class</exclude>
-                                <exclude>**/*StressTest.class</exclude>
-                            </excludes>
-                            <argLine>
-                                ${tests.jvm.argline}
-                            </argLine>
-                            <jvmArgs>
-                                <param>-Xmx512m</param>
-                                <param>-Xss256k</param>
-                                <param>-XX:MaxDirectMemorySize=512m</param>
-                                <param>-Des.logger.prefix=</param>
-                            </jvmArgs>
-                            <shuffleOnSlave>${tests.shuffle}</shuffleOnSlave>
-                            <sysouts>${tests.verbose}</sysouts>
-                            <seed>${tests.seed}</seed>
-                            <haltOnFailure>${tests.failfast}</haltOnFailure>
-                            <systemProperties>
-                                <!-- RandomizedTesting library system properties -->
-                                <tests.jvm.argline>${tests.jvm.argline}</tests.jvm.argline>
-                                <tests.iters>${tests.iters}</tests.iters>
-                                <tests.maxfailures>${tests.maxfailures}</tests.maxfailures>
-                                <tests.failfast>${tests.failfast}</tests.failfast>
-                                <tests.class>${tests.class}</tests.class>
-                                <tests.method>${tests.method}</tests.method>
-                                <tests.nightly>${tests.nightly}</tests.nightly>
-                                <tests.badapples>${tests.badapples}</tests.badapples>
-                                <tests.weekly>${tests.weekly}</tests.weekly>
-                                <tests.slow>${tests.slow}</tests.slow>
-                                <tests.aws>${tests.aws}</tests.aws>
-                                <tests.config>${tests.config}</tests.config>
-                                <tests.awaitsfix>${tests.awaitsfix}</tests.awaitsfix>
-                                <tests.slow>${tests.slow}</tests.slow>
-                                <tests.timeoutSuite>${tests.timeoutSuite}</tests.timeoutSuite>
-                                <tests.showSuccess>${tests.showSuccess}</tests.showSuccess>
-                                <tests.integration>${tests.integration}</tests.integration>
-                                <tests.cluster_seed>${tests.cluster_seed}</tests.cluster_seed>
-                                <tests.client.ratio>${tests.client.ratio}</tests.client.ratio>
-                                <es.logger.level>${es.logger.level}</es.logger.level>
-                                <java.awt.headless>true</java.awt.headless>
-                            </systemProperties>
-                        </configuration>
-                    </execution>
-                </executions>
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
diff --git a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java
index abd4bfaf374..7ed03939b46 100644
--- a/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java
+++ b/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTest.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cloud.aws;
 
 import com.carrotsearch.randomizedtesting.annotations.TestGroup;
+
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -27,6 +28,7 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.FailedToResolveConfigException;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.elasticsearch.test.ElasticsearchIntegrationTest.ThirdParty;
 import org.junit.After;
 import org.junit.Before;
 
@@ -38,8 +40,12 @@ import java.util.HashMap;
 import java.util.Map;
 
 /**
- *
+ * Base class for AWS tests that require credentials.
+ * <p>
+ * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/config}
+ * in order to run these tests.
  */
+@ThirdParty
 public abstract class AbstractAwsTest extends ElasticsearchIntegrationTest {
 
     /**
@@ -71,22 +77,6 @@ public abstract class AbstractAwsTest extends ElasticsearchIntegrationTest {
         }
     }
 
-
-    /**
-     * Annotation for tests that require AWS to run. AWS tests are disabled by default.
-     * Look at README file for details on how to run tests
-     */
-    @Documented
-    @Inherited
-    @Retention(RetentionPolicy.RUNTIME)
-    @TestGroup(enabled = false, sysProperty = SYSPROP_AWS)
-    public @interface AwsTest {
-    }
-
-    /**
-     */
-    public static final String SYSPROP_AWS = "tests.aws";
-
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
                 ImmutableSettings.Builder settings = ImmutableSettings.builder()
@@ -105,10 +95,10 @@ public abstract class AbstractAwsTest extends ElasticsearchIntegrationTest {
             if (Strings.hasText(System.getProperty("tests.config"))) {
                 settings.loadFromUrl(environment.resolveConfig(System.getProperty("tests.config")));
             } else {
-                fail("to run integration tests, you need to set -Dtest.aws=true and -Dtests.config=/path/to/elasticsearch.yml");
+                throw new IllegalStateException("to run integration tests, you need to set -Dtest.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
             }
         } catch (FailedToResolveConfigException exception) {
-            fail("your test configuration file is incorrect: " + System.getProperty("tests.config"));
+            throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
         }
         return settings.build();
     }
diff --git a/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java b/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java
index 13e85280c44..d656d6485f6 100644
--- a/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java
+++ b/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryITest.java
@@ -21,7 +21,6 @@ package org.elasticsearch.discovery.ec2;
 
 
 import org.elasticsearch.cloud.aws.AbstractAwsTest;
-import org.elasticsearch.cloud.aws.AbstractAwsTest.AwsTest;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
@@ -35,7 +34,6 @@ import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilde
  * starting.
  * This test requires AWS to run.
  */
-@AwsTest
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
 public class Ec2DiscoveryITest extends AbstractAwsTest {
 
diff --git a/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsITest.java b/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsITest.java
index ca1b4d4ef19..f3038b6f017 100644
--- a/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsITest.java
+++ b/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryUpdateSettingsITest.java
@@ -22,7 +22,6 @@ package org.elasticsearch.discovery.ec2;
 
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
 import org.elasticsearch.cloud.aws.AbstractAwsTest;
-import org.elasticsearch.cloud.aws.AbstractAwsTest.AwsTest;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
@@ -37,7 +36,6 @@ import static org.hamcrest.CoreMatchers.is;
  * starting.
  * This test requires AWS to run.
  */
-@AwsTest
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
 public class Ec2DiscoveryUpdateSettingsITest extends AbstractAwsTest {
 
diff --git a/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
index b64e2944aa9..c3807745a58 100644
--- a/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
+++ b/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
@@ -23,13 +23,13 @@ import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ClusterAdminClient;
 import org.elasticsearch.cloud.aws.AbstractAwsTest;
-import org.elasticsearch.cloud.aws.AbstractAwsTest.AwsTest;
 import org.elasticsearch.cloud.aws.AwsS3Service;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.settings.ImmutableSettings;
@@ -53,7 +53,6 @@ import static org.hamcrest.Matchers.*;
 
 /**
  */
-@AwsTest
 @ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, transportClientRatio = 0.0)
 abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTest {