diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java
index 0d9a05c4671..dab11583796 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java
@@ -237,8 +237,9 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
         }
         if (name.equals(ScoreSortBuilder.NAME)) {
             sort(SortBuilders.scoreSort().order(order));
+        } else {
+            sort(SortBuilders.fieldSort(name).order(order));
         }
-        sort(SortBuilders.fieldSort(name).order(order));
         return this;
     }
 
@@ -254,8 +255,9 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
         }
         if (name.equals(ScoreSortBuilder.NAME)) {
             sort(SortBuilders.scoreSort());
+        } else {
+            sort(SortBuilders.fieldSort(name));
         }
-        sort(SortBuilders.fieldSort(name));
         return this;
     }
 
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorTests.java
index 585cd7f9ff4..b0879097573 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorTests.java
@@ -49,7 +49,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.metrics.TopHits;
 import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper;
 import org.elasticsearch.search.sort.SortOrder;
 
@@ -207,4 +206,10 @@ public class TopHitsAggregatorTests extends AggregatorTestCase {
         reader.close();
         directory.close();
     }
+
+    public void testSortByScore() throws Exception {
+        // just check that it does not fail with exceptions
+        testCase(new MatchAllDocsQuery(), topHits("_name").sort("_score", SortOrder.DESC));
+        testCase(new MatchAllDocsQuery(), topHits("_name").sort("_score"));
+    }
 }