diff --git a/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java b/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
index 35e2b3a8025..c6cc465b801 100644
--- a/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
+++ b/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
@@ -84,9 +84,7 @@ public abstract class Aggregator implements Releasable, ReaderContextAware {
         assert factories != null : "sub-factories provided to BucketAggregator must not be null, use AggragatorFactories.EMPTY instead";
         this.factories = factories;
         this.subAggregators = factories.createSubAggregators(this, estimatedBucketsCount);
-        // TODO: change it to SEARCH_PHASE, but this would imply allocating the aggregators in the QUERY
-        // phase instead of DFS like it is done today
-        context.searchContext().addReleasable(this, Lifetime.CONTEXT);
+        context.searchContext().addReleasable(this, Lifetime.PHASE);
     }
 
     /**
diff --git a/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 017c2c69482..5f10cd361df 100644
--- a/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -89,11 +89,15 @@ public class QueryPhase implements SearchPhase {
     @Override
     public void preProcess(SearchContext context) {
         context.preProcess();
-        facetPhase.preProcess(context);
-        aggregationPhase.preProcess(context);
     }
 
     public void execute(SearchContext searchContext) throws QueryPhaseExecutionException {
+        // Pre-process facets and aggregations as late as possible. In the case of a DFS_Q_T_F
+        // request, preProcess is called on the DFS phase phase, this is why we pre-process them
+        // here to make sure it happens during the QUERY phase
+        facetPhase.preProcess(searchContext);
+        aggregationPhase.preProcess(searchContext);
+
         searchContext.queryResult().searchTimedOut(false);
 
         searchContext.searcher().inStage(ContextIndexSearcher.Stage.MAIN_QUERY);