diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 8b0b183bb58..516863edc1a 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -114,6 +114,8 @@ Other Changes
 
 * SOLR-7358: TestRestoreCore fails in Windows (Ishan Chattopadhyaya via Varun Thacker)
 
+* SOLR-7371: Make DocSet implement Accountable to estimate memory usage. (yonik, shalin)
+
 ==================  5.1.0 ==================
 
 Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release
diff --git a/solr/core/src/java/org/apache/solr/search/BitDocSet.java b/solr/core/src/java/org/apache/solr/search/BitDocSet.java
index bc72e57ac18..dd0b6ffabac 100644
--- a/solr/core/src/java/org/apache/solr/search/BitDocSet.java
+++ b/solr/core/src/java/org/apache/solr/search/BitDocSet.java
@@ -17,16 +17,21 @@
 
 package org.apache.solr.search;
 
+import java.util.Collection;
+import java.util.Collections;
+
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.BitsFilteredDocIdSet;
 import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Filter;
+import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BitDocIdSet;
 import org.apache.lucene.util.BitSetIterator;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * <code>BitDocSet</code> represents an unordered set of Lucene Document Ids
@@ -35,6 +40,8 @@ import org.apache.lucene.util.FixedBitSet;
  * @since solr 0.9
  */
 public class BitDocSet extends DocSetBase {
+  private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BitDocSet.class);
+
   final FixedBitSet bits;
   int size;    // number of docs in the set (cached for perf)
 
@@ -253,11 +260,6 @@ public class BitDocSet extends DocSetBase {
     return new BitDocSet(newbits);
   }
   
-  @Override
-  public long memSize() {
-    return (bits.getBits().length << 3) + 16;
-  }
-
   @Override
   protected BitDocSet clone() {
     return new BitDocSet(bits.clone(), size);
@@ -359,4 +361,14 @@ public class BitDocSet extends DocSetBase {
       }
     };
   }
+
+  @Override
+  public long ramBytesUsed() {
+    return BASE_RAM_BYTES_USED + bits.ramBytesUsed();
+  }
+
+  @Override
+  public Collection<Accountable> getChildResources() {
+    return Collections.emptyList();
+  }
 }
diff --git a/solr/core/src/java/org/apache/solr/search/DocSet.java b/solr/core/src/java/org/apache/solr/search/DocSet.java
index d918c6dcc3c..de98d07042d 100644
--- a/solr/core/src/java/org/apache/solr/search/DocSet.java
+++ b/solr/core/src/java/org/apache/solr/search/DocSet.java
@@ -20,6 +20,7 @@ package org.apache.solr.search;
 import java.io.Closeable;
 
 import org.apache.lucene.search.Filter;
+import org.apache.lucene.util.Accountable;
 import org.apache.solr.common.SolrException;
 
 /**
@@ -32,7 +33,7 @@ import org.apache.solr.common.SolrException;
  *
  * @since solr 0.9
  */
-public interface DocSet extends Closeable /* extends Collection<Integer> */ {
+public interface DocSet extends Closeable, Accountable /* extends Collection<Integer> */ {
   
   /**
    * Adds the specified document if it is not currently in the DocSet
@@ -77,15 +78,6 @@ public interface DocSet extends Closeable /* extends Collection<Integer> */ {
    */
   public DocIterator iterator();
 
-  /**
-   * Returns the approximate amount of memory taken by this DocSet.
-   * This is only an approximation and doesn't take into account java object overhead.
-   *
-   * @return
-   * the approximate memory consumption in bytes
-   */
-  public long memSize();
-
   /**
    * Returns the intersection of this set with another set.  Neither set is modified - a new DocSet is
    * created and returned.
diff --git a/solr/core/src/java/org/apache/solr/search/DocSlice.java b/solr/core/src/java/org/apache/solr/search/DocSlice.java
index 00a0ebc0e89..a3500d02866 100644
--- a/solr/core/src/java/org/apache/solr/search/DocSlice.java
+++ b/solr/core/src/java/org/apache/solr/search/DocSlice.java
@@ -18,6 +18,11 @@
 package org.apache.solr.search;
 
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * <code>DocSlice</code> implements DocList as an array of docids and optional scores.
@@ -26,6 +31,8 @@ import java.util.Arrays;
  * @since solr 0.9
  */
 public class DocSlice extends DocSetBase implements DocList {
+  private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DocSlice.class);
+
   final int offset;    // starting position of the docs (zero based)
   final int len;       // number of positions used in arrays
   final int[] docs;    // a slice of documents (docs 0-100 of the query)
@@ -34,6 +41,8 @@ public class DocSlice extends DocSetBase implements DocList {
   final int matches;
   final float maxScore;
 
+  final long ramBytesUsed;
+
   /**
    * Primary constructor for a DocSlice instance.
    *
@@ -50,6 +59,7 @@ public class DocSlice extends DocSetBase implements DocList {
     this.scores=scores;
     this.matches=matches;
     this.maxScore=maxScore;
+    this.ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(docs) + RamUsageEstimator.sizeOf(scores);
   }
 
   @Override
@@ -85,14 +95,6 @@ public class DocSlice extends DocSetBase implements DocList {
   public int matches() { return matches; }
 
 
-  @Override
-  public long memSize() {
-    return (docs.length<<2)
-            + (scores==null ? 0 : (scores.length<<2))
-            + 24;
-  }
-
-
   @Override
   public boolean exists(int doc) {
     int end = offset+len;
@@ -175,4 +177,14 @@ public class DocSlice extends DocSetBase implements DocList {
     } catch (CloneNotSupportedException e) {}
     return null;
   }
+
+  @Override
+  public long ramBytesUsed() {
+    return ramBytesUsed;
+  }
+
+  @Override
+  public Collection<Accountable> getChildResources() {
+    return Collections.emptyList();
+  }
 }
diff --git a/solr/core/src/java/org/apache/solr/search/HashDocSet.java b/solr/core/src/java/org/apache/solr/search/HashDocSet.java
index 28ed05abe01..892bee6eadc 100644
--- a/solr/core/src/java/org/apache/solr/search/HashDocSet.java
+++ b/solr/core/src/java/org/apache/solr/search/HashDocSet.java
@@ -17,7 +17,12 @@
 
 package org.apache.solr.search;
 
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BitUtil;
+import org.apache.lucene.util.RamUsageEstimator;
 
 
 /**
@@ -30,6 +35,8 @@ import org.apache.lucene.util.BitUtil;
  * @since solr 0.9
  */
 public final class HashDocSet extends DocSetBase {
+  private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(HashDocSet.class);
+
   /** Default load factor to use for HashDocSets.  We keep track of the inverse
    *  since multiplication is so much faster than division.  The default
    *  is 1.0f / 0.75f
@@ -48,10 +55,13 @@ public final class HashDocSet extends DocSetBase {
 
   private final int mask;
 
+  private final long ramBytesUsed;
+
   public HashDocSet(HashDocSet set) {
     this.table = set.table.clone();
     this.size = set.size;
     this.mask = set.mask;
+    this.ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(table);
   }
 
   /** Create a HashDocSet from a list of *unique* ids */
@@ -79,6 +89,8 @@ public final class HashDocSet extends DocSetBase {
     }
 
     size = len;
+
+    ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(table);
   }
 
   void put(int doc) {
@@ -155,11 +167,6 @@ public final class HashDocSet extends DocSetBase {
     };
   }
 
-  @Override
-  public long memSize() {
-    return (table.length<<2) + 20;
-  }
-
   @Override
   public DocSet intersection(DocSet other) {
    if (other instanceof HashDocSet) {
@@ -296,4 +303,15 @@ public final class HashDocSet extends DocSetBase {
 
   // don't implement andNotSize() and unionSize() on purpose... they are implemented
   // in BaseDocSet in terms of intersectionSize().
+
+
+  @Override
+  public long ramBytesUsed() {
+    return ramBytesUsed;
+  }
+
+  @Override
+  public Collection<Accountable> getChildResources() {
+    return Collections.emptyList();
+  }
 }
diff --git a/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java b/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
index db4bd1c266d..33f58728f87 100644
--- a/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
+++ b/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java
@@ -17,12 +17,16 @@
 
 package org.apache.solr.search;
 
+import java.util.Collection;
+import java.util.Collections;
+
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.BitsFilteredDocIdSet;
 import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Filter;
+import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.RamUsageEstimator;
@@ -31,13 +35,17 @@ import org.apache.lucene.util.RamUsageEstimator;
  * <code>SortedIntDocSet</code> represents a sorted set of Lucene Document Ids.
  */
 public class SortedIntDocSet extends DocSetBase {
+  private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SortedIntDocSet.class);
+
   protected final int[] docs;
+  protected final long ramBytesUsed;
 
   /**
    * @param docs  Sorted list of ids
    */
   public SortedIntDocSet(int[] docs) {
     this.docs = docs;
+    this.ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(docs);
     // if (firstNonSorted(docs,0,docs.length)>=0) throw new RuntimeException("NON SORTED DOCS!!!");
   }
 
@@ -54,11 +62,6 @@ public class SortedIntDocSet extends DocSetBase {
   @Override
   public int size()      { return docs.length; }
 
-  @Override
-  public long memSize() {
-    return (docs.length<<2)+8;
-  }
-
   public static int[] zeroInts = new int[0];
   public static SortedIntDocSet zero = new SortedIntDocSet(zeroInts);
 
@@ -785,4 +788,14 @@ public class SortedIntDocSet extends DocSetBase {
   protected SortedIntDocSet clone() {
     return new SortedIntDocSet(docs.clone());
   }
+
+  @Override
+  public long ramBytesUsed() {
+    return ramBytesUsed;
+  }
+
+  @Override
+  public Collection<Accountable> getChildResources() {
+    return Collections.emptyList();
+  }
 }
diff --git a/solr/core/src/test/org/apache/solr/search/DocSetPerf.java b/solr/core/src/test/org/apache/solr/search/DocSetPerf.java
index 37c13df4c97..711c8fff213 100644
--- a/solr/core/src/test/org/apache/solr/search/DocSetPerf.java
+++ b/solr/core/src/test/org/apache/solr/search/DocSetPerf.java
@@ -74,7 +74,7 @@ public class DocSetPerf {
     String test = args[3].intern();
     int iter = Integer.parseInt(args[4]);
 
-    int ret=0;
+    long ret=0;
 
     FixedBitSet[] sets = new FixedBitSet[numSets];
     DocSet[] bset = new DocSet[numSets];
@@ -153,7 +153,7 @@ public class DocSetPerf {
 
         if (oper=="intersect") {
           DocSet res = a.intersection(b);
-          ret += res.memSize();
+          ret += res.ramBytesUsed();
         } else if (oper=="intersectSize") {
           ret += a.intersectionSize(b);
         } else if (oper=="intersectAndSize") {