diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java
index d47610c52ad..c639e9fc376 100644
--- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java
+++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java
@@ -122,22 +122,25 @@ public class PrecisionAtN implements RankedListQualityMetric {
      **/
     @Override
     public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List<RatedDocument> ratedDocs) {
-        int good = 0;
-        int bad = 0;
+        int truePositives = 0;
+        int falsePositives = 0;
         List<RatedSearchHit> ratedSearchHits = joinHitsWithRatings(hits, ratedDocs);
         for (RatedSearchHit hit : ratedSearchHits) {
             Optional<Integer> rating = hit.getRating();
             if (rating.isPresent()) {
                 if (rating.get() >= this.relevantRatingThreshhold) {
-                    good++;
+                    truePositives++;
                 } else {
-                    bad++;
+                    falsePositives++;
                 }
             }
         }
-        double precision = (double) good / (good + bad);
+        double precision = 0.0;
+        if (truePositives + falsePositives > 0) {
+            precision = (double) truePositives / (truePositives + falsePositives);
+        }
         EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision);
-        evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(good, good + bad));
+        evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(truePositives, truePositives + falsePositives));
         evalQueryQuality.addHitsAndRatings(ratedSearchHits);
         return evalQueryQuality;
     }
diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java
index b65a860990e..800a38c0ec9 100644
--- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java
+++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java
@@ -35,11 +35,10 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Vector;
-import java.util.concurrent.ExecutionException;
 
 public class PrecisionAtNTests extends ESTestCase {
 
-    public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException {
+    public void testPrecisionAtFiveCalculation() {
         List<RatedDocument> rated = new ArrayList<>();
         rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal()));
         EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated, "test", "testtype"), rated);
@@ -48,7 +47,7 @@ public class PrecisionAtNTests extends ESTestCase {
         assertEquals(1, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved());
     }
 
-    public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException {
+    public void testPrecisionAtFiveIgnoreOneResult() {
         List<RatedDocument> rated = new ArrayList<>();
         rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal()));
         rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal()));
@@ -65,7 +64,7 @@ public class PrecisionAtNTests extends ESTestCase {
      * test that the relevant rating threshold can be set to something larger than 1.
      * e.g. we set it to 2 here and expect dics 0-2 to be not relevant, doc 3 and 4 to be relevant
      */
-    public void testPrecisionAtFiveRelevanceThreshold() throws IOException, InterruptedException, ExecutionException {
+    public void testPrecisionAtFiveRelevanceThreshold() {
         List<RatedDocument> rated = new ArrayList<>();
         rated.add(new RatedDocument("test", "testtype", "0", 0));
         rated.add(new RatedDocument("test", "testtype", "1", 1));
@@ -80,7 +79,7 @@ public class PrecisionAtNTests extends ESTestCase {
         assertEquals(5, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved());
     }
 
-    public void testPrecisionAtFiveCorrectIndex() throws IOException, InterruptedException, ExecutionException {
+    public void testPrecisionAtFiveCorrectIndex() {
         List<RatedDocument> rated = new ArrayList<>();
         rated.add(new RatedDocument("test_other", "testtype", "0", Rating.RELEVANT.ordinal()));
         rated.add(new RatedDocument("test_other", "testtype", "1", Rating.RELEVANT.ordinal()));
@@ -94,7 +93,7 @@ public class PrecisionAtNTests extends ESTestCase {
         assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved());
     }
 
-    public void testPrecisionAtFiveCorrectType() throws IOException, InterruptedException, ExecutionException {
+    public void testPrecisionAtFiveCorrectType() {
         List<RatedDocument> rated = new ArrayList<>();
         rated.add(new RatedDocument("test", "other_type", "0", Rating.RELEVANT.ordinal()));
         rated.add(new RatedDocument("test", "other_type", "1", Rating.RELEVANT.ordinal()));
@@ -107,6 +106,14 @@ public class PrecisionAtNTests extends ESTestCase {
         assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved());
     }
 
+    public void testNoRatedDocs() throws Exception {
+        List<RatedDocument> rated = new ArrayList<>();
+        EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated, "test", "testtype"), rated);
+        assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001);
+        assertEquals(0, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
+        assertEquals(0, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved());
+    }
+
     public void testParseFromXContent() throws IOException {
         String xContent = " {\n"
          + "   \"size\": 10,\n"