diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e6034567a7f..1130f5272ed 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -110,6 +110,9 @@ Bug Fixes
 * SOLR-11231: Guard against unset fields when performing language detection.
   (Chris Beer via Steve Rowe)
 
+* SOLR-11553: JSON Facet API: facet refinement could use UIF in phase one and DV uninversion in phase 2, resulting
+  in too much memory use. (yonik)
+
 Optimizations
 ----------------------
 * SOLR-11285: Refactor autoscaling framework to avoid direct references to Zookeeper and Solr
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java
index 578994f7be5..1a595491a3a 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetField.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetField.java
@@ -101,6 +101,11 @@ public class FacetField extends FacetRequestSorted {
 
     if (fcontext.facetInfo != null) {
       // refinement... we will end up either skipping the entire facet, or doing calculating only specific facet buckets
+      if (multiToken && !sf.hasDocValues() && method!=FacetMethod.DV) {
+        // Match the access method from the first phase.
+        // It won't always matter, but does currently for an all-values bucket
+        return new FacetFieldProcessorByArrayUIF(fcontext, this, sf);
+      }
       return new FacetFieldProcessorByArrayDV(fcontext, this, sf);
     }
 
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java b/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java
index 1754175cb3a..c9f3b0b78d7 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UnInvertedField.java
@@ -603,4 +603,16 @@ public class UnInvertedField extends DocTermOrds {
 
     return uif;
   }
+
+  // Returns null if not already populated
+  public static UnInvertedField checkUnInvertedField(String field, SolrIndexSearcher searcher) throws IOException {
+    SolrCache<String, UnInvertedField> cache = searcher.getFieldValueCache();
+    if (cache == null) {
+      return null;
+    }
+    UnInvertedField uif = cache.get(field);  // cache is already synchronized, so no extra sync needed
+    // placeholder is an implementation detail, keep it hidden and return null if that is what we got
+    return uif==uifPlaceholder ? null : uif;
+  }
+
 }