diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 16953e5edc5..e8960088e28 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -85,6 +85,9 @@ Bug Fixes
 * LUCENE-6415: TermsQuery.extractTerms is a no-op (used to throw an
   UnsupportedOperationException). (Adrien Grand)
 
+* LUCENE-6416: BooleanQuery.extractTerms now only extracts terms from scoring
+  clauses. (Adrien Grand)
+
 API Changes
 
 * LUCENE-6377: SearcherFactory#newSearcher now accepts the previous reader
diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java b/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
index c3d2f566bca..d96613a165a 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
@@ -237,7 +237,7 @@ public class BooleanQuery extends Query implements Iterable<BooleanClause> {
   @Override
   public void extractTerms(Set<Term> terms) {
     for (BooleanClause clause : clauses) {
-      if (clause.isProhibited() == false) {
+      if (clause.isScoring()) {
         clause.getQuery().extractTerms(terms);
       }
     }
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java
index ec99739b253..f1b299d5e95 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java
@@ -21,7 +21,9 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.BitSet;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -755,4 +757,19 @@ public class TestBooleanQuery extends LuceneTestCase {
     bq.add(new TermQuery(new Term("field", "d")), Occur.FILTER);
     assertEquals("a +b -c #d", bq.toString("field"));
   }
+
+  public void testExtractTerms() {
+    Term a = new Term("f", "a");
+    Term b = new Term("f", "b");
+    Term c = new Term("f", "c");
+    Term d = new Term("f", "d");
+    BooleanQuery bq = new BooleanQuery();
+    bq.add(new TermQuery(a), Occur.SHOULD);
+    bq.add(new TermQuery(b), Occur.MUST);
+    bq.add(new TermQuery(c), Occur.FILTER);
+    bq.add(new TermQuery(d), Occur.MUST_NOT);
+    Set<Term> terms = new HashSet<>();
+    bq.extractTerms(terms);
+    assertEquals(new HashSet<>(Arrays.asList(a, b)), terms);
+  }
 }