diff --git a/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index b0a653eedbf..302ff9290e6 100644
--- a/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/src/test/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -31,6 +31,8 @@ import org.elasticsearch.search.suggest.Suggest;
 import org.hamcrest.Matcher;
 
 import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.*;
@@ -46,6 +48,16 @@ public class ElasticsearchAssertions {
     public static void assertHitCount(SearchResponse searchResponse, long expectedHitCount) {
         assertThat(searchResponse.getHits().totalHits(), is(expectedHitCount));
     }
+    
+    public static void assertSearchHits(SearchResponse searchResponse, String... ids) {
+        assertThat("Expected different hit count", searchResponse.getHits().hits().length, equalTo(ids.length));
+
+        Set<String> idsSet = new HashSet<String>(Arrays.asList(ids));
+        for (SearchHit hit : searchResponse.getHits()) {
+            assertThat("Expected id: " + hit.getId() + " in the result but wasn't", idsSet.remove(hit.getId()), equalTo(true));
+        }
+        assertThat("Expected ids: " + Arrays.toString(idsSet.toArray(new String[0])) + " in the result - result size differs", idsSet.size(), equalTo(0));
+    }
 
     public static void assertHitCount(CountResponse countResponse, long expectedHitCount) {
         assertThat(countResponse.getCount(), is(expectedHitCount));
diff --git a/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java b/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java
index e4f3eadf104..254b7f9ba3f 100644
--- a/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java
+++ b/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java
@@ -39,13 +39,9 @@ import java.util.Arrays;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.FilterBuilders.*;
 import static org.elasticsearch.index.query.QueryBuilders.*;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.hamcrest.MatcherAssert.assertThat;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 /**
  *
@@ -545,8 +541,8 @@ public class SimpleQueryTests extends AbstractSharedClusterTest {
                 .execute().actionGet();
 
         assertThat(searchResponse.getHits().totalHits(), equalTo(2l));
-        assertThat("1", equalTo(searchResponse.getHits().getAt(0).id()));
-        assertThat("2", equalTo(searchResponse.getHits().getAt(1).id()));
+        // this uses dismax so scores are equal and the order can be arbitrary
+        assertSearchHits(searchResponse, "1", "2");
 
         builder.useDisMax(false);
         searchResponse = client().prepareSearch()