diff --git a/src/test/java/org/elasticsearch/mlt/MoreLikeThisActionTests.java b/src/test/java/org/elasticsearch/mlt/MoreLikeThisActionTests.java
index 1ea485a3c43..1bc927b032a 100644
--- a/src/test/java/org/elasticsearch/mlt/MoreLikeThisActionTests.java
+++ b/src/test/java/org/elasticsearch/mlt/MoreLikeThisActionTests.java
@@ -401,47 +401,50 @@ public class MoreLikeThisActionTests extends ElasticsearchIntegrationTest {
             builders.add(client().prepareIndex("test", "type1").setSource("text", texts[i]).setId(String.valueOf(i)));
         }
         indexRandom(true, builders);
+        int iters = between(10, 20);
+        for (int j = 0; j < iters; j++) {
+            logger.info("Running MoreLikeThis DSL with IDs");
+            String id = String.valueOf(getRandom().nextInt(texts.length));
+            Client client = client();
+            MoreLikeThisQueryBuilder queryBuilder = QueryBuilders.moreLikeThisQuery("text").ids(id).minTermFreq(1).minDocFreq(1);
+            SearchResponse mltResponseDSL = client.prepareSearch()
+                    .setSearchType(SearchType.QUERY_THEN_FETCH)
+                    .setTypes("type1")
+                    .setQuery(queryBuilder)
+                    .setSize(texts.length)
+                    .execute().actionGet();
+            assertSearchResponse(mltResponseDSL);
 
-        logger.info("Running MoreLikeThis DSL with IDs");
-        Client client = client();
-        MoreLikeThisQueryBuilder queryBuilder = QueryBuilders.moreLikeThisQuery("text").ids("0").minTermFreq(1).minDocFreq(1);
-        SearchResponse mltResponseDSL = client.prepareSearch()
-                .setSearchType(SearchType.QUERY_THEN_FETCH)
-                .setTypes("type1")
-                .setQuery(queryBuilder)
-                .setSize(texts.length)
-                .execute().actionGet();
-        assertSearchResponse(mltResponseDSL);
+            logger.info("Running MoreLikeThis API");
+            MoreLikeThisRequest mltRequest = moreLikeThisRequest("test").type("type1").searchSize(texts.length).id(id).minTermFreq(1).minDocFreq(1);
+            SearchResponse mltResponseAPI = client.moreLikeThis(mltRequest).actionGet();
+            assertSearchResponse(mltResponseAPI);
 
-        logger.info("Running MoreLikeThis API");
-        MoreLikeThisRequest mltRequest = moreLikeThisRequest("test").type("type1").searchSize(texts.length).id("0").minTermFreq(1).minDocFreq(1);
-        SearchResponse mltResponseAPI = client.moreLikeThis(mltRequest).actionGet();
-        assertSearchResponse(mltResponseAPI);
+            logger.info("Ensure the documents and scores returned are the same.");
+            SearchHit[] hitsDSL = mltResponseDSL.getHits().hits();
+            SearchHit[] hitsAPI = mltResponseAPI.getHits().hits();
 
-        logger.info("Ensure the documents and scores returned are the same.");
-        SearchHit[] hitsDSL = mltResponseDSL.getHits().hits();
-        SearchHit[] hitsAPI = mltResponseAPI.getHits().hits();
+            // we have to resort since the results might come from
+            // different shards and docIDs that are used for tie-breaking might not be the same on the shards
+            Comparator<SearchHit> cmp = new Comparator<SearchHit>() {
 
-        // we have to resort since the results might come from
-        // different shards and docIDs that are used for tie-breaking might not be the same on the shards
-        Comparator<SearchHit> cmp = new Comparator<SearchHit>() {
-
-            @Override
-            public int compare(SearchHit o1, SearchHit o2) {
-                if (Float.compare(o1.getScore(), o2.getScore()) == 0) {
-                    return o1.getId().compareTo(o2.getId());
+                @Override
+                public int compare(SearchHit o1, SearchHit o2) {
+                    if (Float.compare(o1.getScore(), o2.getScore()) == 0) {
+                        return o1.getId().compareTo(o2.getId());
+                    }
+                    return Float.compare(o1.getScore(), o2.getScore());
                 }
-                return Float.compare(o1.getScore(), o2.getScore());
+            };
+            ArrayUtil.timSort(hitsDSL, cmp);
+            ArrayUtil.timSort(hitsAPI, cmp);
+            assertThat("Not the same number of results.", hitsAPI.length, equalTo(hitsDSL.length));
+            for (int i = 0; i < hitsDSL.length; i++) {
+                assertThat("Expected id: " + hitsDSL[i].getId() + " at position " + i + " but wasn't.",
+                        hitsAPI[i].getId(), equalTo(hitsDSL[i].getId()));
+                assertThat("Expected score: " + hitsDSL[i].getScore() + " at position " + i + " but wasn't.",
+                        hitsAPI[i].getScore(), equalTo(hitsDSL[i].getScore()));
             }
-        };
-        ArrayUtil.timSort(hitsDSL, cmp);
-        ArrayUtil.timSort(hitsAPI, cmp);
-        assertThat("Not the same number of results.", hitsAPI.length, equalTo(hitsDSL.length));
-        for (int i = 0; i < hitsDSL.length; i++) {
-            assertThat("Expected id: " + hitsDSL[i].getId() + " at position " + i + " but wasn't.",
-                    hitsAPI[i].getId(), equalTo(hitsDSL[i].getId()));
-            assertThat("Expected score: " + hitsDSL[i].getScore() + " at position " + i + " but wasn't.",
-                    hitsAPI[i].getScore(), equalTo(hitsDSL[i].getScore()));
         }
     }