diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java
index a46a3f58eab..f97ba7c7975 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java
@@ -111,7 +111,7 @@ public class PrecisionAtN extends RankedListQualityMetric {
 
         int good = 0;
         int bad = 0;
-        Collection<RatedDocumentKey> unknownDocIds = new ArrayList<RatedDocumentKey>();
+        Collection<RatedDocumentKey> unknownDocIds = new ArrayList<>();
         for (int i = 0; (i < n && i < hits.length); i++) {
             RatedDocumentKey hitKey = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId());
             if (relevantDocIds.contains(hitKey)) {
diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java
index 4162896ef7d..70ec185fcbe 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java
@@ -95,21 +95,21 @@ public class ReciprocalRank extends RankedListQualityMetric {
      **/
     @Override
     public EvalQueryQuality evaluate(SearchHit[] hits, List<RatedDocument> ratedDocs) {
-        Set<String> relevantDocIds = new HashSet<>();
-        Set<String> irrelevantDocIds = new HashSet<>();
+        Set<RatedDocumentKey> relevantDocIds = new HashSet<>();
+        Set<RatedDocumentKey> irrelevantDocIds = new HashSet<>();
         for (RatedDocument doc : ratedDocs) {
             if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) {
-                relevantDocIds.add(doc.getDocID());
+                relevantDocIds.add(doc.getKey());
             } else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) {
-                irrelevantDocIds.add(doc.getDocID());
+                irrelevantDocIds.add(doc.getKey());
             }
         }
 
-        Collection<String> unknownDocIds = new ArrayList<>();
+        Collection<RatedDocumentKey> unknownDocIds = new ArrayList<>();
         int firstRelevant = -1;
         boolean found = false;
         for (int i = 0; i < hits.length; i++) {
-            String id = hits[i].getId();
+            RatedDocumentKey id = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId());
             if (relevantDocIds.contains(id)) {
                 if (found == false && i < maxAcceptableRank) {
                     firstRelevant = i + 1; // add one because rank is not
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java
index 12dd808cff7..1abc671606c 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java
@@ -20,8 +20,9 @@
 package org.elasticsearch.index.rankeval;
 
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.rankeval.PrecisionAtN.Rating;
-import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.test.ESTestCase;
 
@@ -42,17 +43,22 @@ public class ReciprocalRankTests extends ESTestCase {
         reciprocalRank.setMaxAcceptableRank(maxRank);
         assertEquals(maxRank, reciprocalRank.getMaxAcceptableRank());
 
-        SearchHit[] hits = new SearchHit[10];
+        InternalSearchHit[] hits = new InternalSearchHit[10];
         for (int i = 0; i < 10; i++) {
             hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap());
+            hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0));
         }
         List<RatedDocument> ratedDocs = new ArrayList<>();
         int relevantAt = 5;
         for (int i = 0; i < 10; i++) {
             if (i == relevantAt) {
-                ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.RELEVANT.ordinal()));
+                ratedDocs.add(new RatedDocument(
+                        new RatedDocumentKey("test", "type", Integer.toString(i)),
+                        Rating.RELEVANT.ordinal()));
             } else {
-                ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.IRRELEVANT.ordinal()));
+                ratedDocs.add(new RatedDocument(
+                        new RatedDocumentKey("test", "type", Integer.toString(i)),
+                        Rating.IRRELEVANT.ordinal()));
             }
         }
 
@@ -67,18 +73,23 @@ public class ReciprocalRankTests extends ESTestCase {
 
     public void testEvaluationOneRelevantInResults() {
         ReciprocalRank reciprocalRank = new ReciprocalRank();
-        SearchHit[] hits = new SearchHit[10];
+        InternalSearchHit[] hits = new InternalSearchHit[10];
         for (int i = 0; i < 10; i++) {
             hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap());
+            hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0));
         }
         List<RatedDocument> ratedDocs = new ArrayList<>();
         // mark one of the ten docs relevant
         int relevantAt = randomIntBetween(0, 9);
         for (int i = 0; i <= 20; i++) {
             if (i == relevantAt) {
-                ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.RELEVANT.ordinal()));
+                ratedDocs.add(new RatedDocument(
+                        new RatedDocumentKey("test", "type", Integer.toString(i)),
+                        Rating.RELEVANT.ordinal()));
             } else {
-                ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.IRRELEVANT.ordinal()));
+                ratedDocs.add(new RatedDocument(
+                        new RatedDocumentKey("test", "type", Integer.toString(i)),
+                        Rating.IRRELEVANT.ordinal()));
             }
         }
 
@@ -97,9 +108,10 @@ public class ReciprocalRankTests extends ESTestCase {
 
     public void testEvaluationNoRelevantInResults() {
         ReciprocalRank reciprocalRank = new ReciprocalRank();
-        SearchHit[] hits = new SearchHit[10];
+        InternalSearchHit[] hits = new InternalSearchHit[10];
         for (int i = 0; i < 10; i++) {
             hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap());
+            hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0));
         }
         List<RatedDocument> ratedDocs = new ArrayList<>();
         EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs);
diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml
index 9eed83336c4..7dd41141b33 100644
--- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml
+++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml
@@ -114,13 +114,13 @@
                 "id": "amsterdam_query",
                 "request": { "query": { "match" : {"text" : "amsterdam" }}},
                 # doc4 should be returned in third position, so reciprocal rank is 1/3
-                "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": : 1}]
+                "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}]
             },
             {
                 "id" : "berlin_query",
                 "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
                 # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 
-                "ratings": [{"key": {"index": "foo", "type": "bar": "doc_id": "doc4"}, "rating": 1}]
+                "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}]
             }
           ],
           "metric" : { "reciprocal_rank": {} }
@@ -139,13 +139,13 @@
               "id": "amsterdam_query",
               "request": { "query": { "match" : {"text" : "amsterdam" }}},
               # doc4 should be returned in third position, so reciprocal rank is 1/3
-              "ratings": [{"key": {"index": "foo", "type": "bar": "doc_id": ""doc4"}, "rating": 1}]
+              "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}]
           },
           {
               "id" : "berlin_query",
               "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
               # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 
-              "ratings": [{"key": {"index": "foo", "type": "bar": "doc_id": "doc4"}, "rating": 1}]
+              "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}]
           }
         ],
         "metric" : {