diff --git a/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java b/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
index 395e2392562..a65f8f07d27 100644
--- a/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
+++ b/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
@@ -172,15 +172,19 @@ public class SearchPhaseController extends AbstractComponent {
                 }
             }
             if (canOptimize) {
-                ScoreDoc[] scoreDocs = result.topDocs().scoreDocs;
-                if (scoreDocs.length == 0) {
-                    return EMPTY_DOCS;
-                }
-                int resultDocsSize = scoreDocs.length < result.size() ? scoreDocs.length : result.size();
                 int offset = result.from();
                 if (scrollSort) {
                     offset = 0;
                 }
+                ScoreDoc[] scoreDocs = result.topDocs().scoreDocs;
+                if (scoreDocs.length == 0 || scoreDocs.length < offset) {
+                    return EMPTY_DOCS;
+                }
+
+                int resultDocsSize = result.size();
+                if ((scoreDocs.length - offset) < resultDocsSize) {
+                    resultDocsSize = scoreDocs.length - offset;
+                }
                 ScoreDoc[] docs = new ScoreDoc[resultDocsSize];
                 for (int i = 0; i < resultDocsSize; i++) {
                     ScoreDoc scoreDoc = scoreDocs[offset + i];