Merge remote-tracking branch 'origin/master' into gradle-master

dev-tools/doap/lucene.rdf

@@ -67,6 +67,13 @@
     </maintainer>
 
     <!-- NOTE: please insert releases in numeric order, NOT chronologically. -->
+   <release>
+      <Version>
+        <name>lucene-8.4.0</name>
+        <created>2019-12-29</created>
+        <revision>8.4.0</revision>
+      </Version>
+    </release>
    <release>
       <Version>
         <name>lucene-8.3.1</name>

dev-tools/doap/solr.rdf

@@ -67,6 +67,13 @@
     </maintainer>
 
     <!-- NOTE: please insert releases in numeric order, NOT chronologically. -->
+    <release>
+      <Version>
+        <name>solr-8.4.0</name>
+        <created>2019-12-29</created>
+        <revision>8.4.0</revision>
+      </Version>
+   </release>
     <release>
       <Version>
         <name>solr-8.3.1</name>

lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java

@@ -298,7 +298,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
     "8.3.0-cfs",
     "8.3.0-nocfs",
     "8.3.1-cfs",
-    "8.3.1-nocfs"
+    "8.3.1-nocfs",
+    "8.4.0-cfs",
+    "8.4.0-nocfs"
   };
 
   public static String[] getOldNames() {
@@ -311,7 +313,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
     "sorted.8.1.1",
     "sorted.8.2.0",
     "sorted.8.3.0",
-    "sorted.8.3.1"
+    "sorted.8.3.1",
+    "sorted.8.4.0"
   };
 
   public static String[] getOldSortedNames() {

solr/CHANGES.txt

@@ -150,6 +150,8 @@ Upgrade Notes
   If you prefer to keep the old (but insecure) serialization strategy, you can start your nodes using the
   property: `-Dsolr.useUnsafeOverseerResponse=true`. Keep in mind that this will be removed in future version of Solr.
  
+ * SOLR-13808: add cache=false into uderneath BoolQParser's filter clause or {"bool":{"filter":..}} to avoid caching in
+   filterCache. (Mikhail Khludnev)
 
 New Features
 ---------------------
@@ -178,6 +180,8 @@ Improvements
 * SOLR-13984: Java's SecurityManager sandbox can be enabled via environment variable,
   SOLR_SECURITY_MANAGER_ENABLED=true. (rmuir)
 
+* SOLR-13808: filter in BoolQParser and {"bool":{"filter":..}} in Query DSL are cached by default (Mikhail Khludnev)
+
 Optimizations
 ---------------------
 (No changes)

solr/bin/solr.in.cmd

@@ -141,8 +141,8 @@ REM SSL Certificates contain host/ip "peer name" information that is validated b
 REM this to false can be useful to disable these checks when re-using a certificate on many hosts
 REM set SOLR_SSL_CHECK_PEER_NAME=true
 REM Override Key/Trust Store types if necessary
-REM set SOLR_SSL_KEY_STORE_TYPE=JKS
+REM set SOLR_SSL_KEY_STORE_TYPE=PKCS12
-REM set SOLR_SSL_TRUST_STORE_TYPE=JKS
+REM set SOLR_SSL_TRUST_STORE_TYPE=PKCS12
 
 REM Uncomment if you want to override previously defined SSL values for HTTP client
 REM otherwise keep them commented and the above values will automatically be set for HTTP clients

solr/bin/solr.in.sh

@@ -158,8 +158,8 @@
 # this to false can be useful to disable these checks when re-using a certificate on many hosts
 #SOLR_SSL_CHECK_PEER_NAME=true
 # Override Key/Trust Store types if necessary
-#SOLR_SSL_KEY_STORE_TYPE=JKS
+#SOLR_SSL_KEY_STORE_TYPE=PKCS12
-#SOLR_SSL_TRUST_STORE_TYPE=JKS
+#SOLR_SSL_TRUST_STORE_TYPE=PKCS12
 
 # Uncomment if you want to override previously defined SSL values for HTTP client
 # otherwise keep them commented and the above values will automatically be set for HTTP clients

solr/core/src/java/org/apache/solr/search/BoolQParserPlugin.java

@@ -21,6 +21,7 @@ import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Query;
 import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.query.FilterQuery;
 import org.apache.solr.request.SolrQueryRequest;
 
 /**
@@ -49,7 +50,18 @@ public class BoolQParserPlugin extends QParserPlugin {
       private void addQueries(BooleanQuery.Builder builder, String[] subQueries, BooleanClause.Occur occur) throws SyntaxError {
         if (subQueries != null) {
           for (String subQuery : subQueries) {
-            builder.add(subQuery(subQuery, null).parse(), occur);
+            final QParser subParser = subQuery(subQuery, null);
+            Query extQuery;
+            if (BooleanClause.Occur.FILTER.equals(occur)) {
+              extQuery = subParser.getQuery();
+              if (!(extQuery instanceof ExtendedQuery) || (
+                  ((ExtendedQuery) extQuery).getCache())) {
+                  extQuery = new FilterQuery(extQuery);
+              }
+            } else {
+              extQuery = subParser.parse();
+            }
+            builder.add(extQuery, occur);
           }
         }
       }

solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java

@@ -16,11 +16,17 @@
  */
 package org.apache.solr.search.json;
 
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.solr.JSONTestUtil;
 import org.apache.solr.SolrTestCaseHS;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.CaffeineCache;
+import org.apache.solr.search.DocSet;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -31,6 +37,7 @@ public class TestJsonRequest extends SolrTestCaseHS {
 
   private static SolrInstances servers;  // for distributed testing
 
+  @SuppressWarnings("deprecation")
   @BeforeClass
   public static void beforeTests() throws Exception {
     systemSetPropertySolrDisableShardsWhitelist("true");
@@ -44,6 +51,7 @@ public class TestJsonRequest extends SolrTestCaseHS {
     }
   }
 
+  @SuppressWarnings("deprecation")
   @AfterClass
   public static void afterTests() throws Exception {
     JSONTestUtil.failRepeatedKeys = false;
@@ -303,17 +311,9 @@ public class TestJsonRequest extends SolrTestCaseHS {
         , "response/numFound==1"
     );
 
-    client.testJQ( params("json","{ " +
+    assertCatANot1(client, "must");
-            " query : {" +
+    
-            "  bool : {" +
+    testFilterCachingLocally(client);
-            "   must : '{!lucene q.op=AND df=cat_s}A'" +
-            "   must_not : '{!lucene v=\\'id:1\\'}'" +
-            "  }" +
-            " }" +
-            "}")
-        , "response/numFound==1"
-    );
-
 
     client.testJQ( params("json","{" +
             " query : '*:*'," +
@@ -407,6 +407,64 @@ public class TestJsonRequest extends SolrTestCaseHS {
 
   }
 
+  private static void testFilterCachingLocally(Client client) throws Exception {
+    if(client.getClientProvider()==null) {
+      final SolrQueryRequest request = req();
+      try {
+        final CaffeineCache<Query,DocSet> filterCache = (CaffeineCache<Query,DocSet>) request.getSearcher().getFilterCache();
+        filterCache.clear();
+        final TermQuery catA = new TermQuery(new Term("cat_s", "A"));
+        assertNull("cache is empty",filterCache.get(catA));
+
+        if(random().nextBoolean()) {
+          if(random().nextBoolean()) {
+            if(random().nextBoolean()) {
+              assertCatANot1(client, "must");
+            }else {
+              assertCatANot1(client, "must", "cat_s:A");
+            }
+          } else {
+            assertCatANot1(client, "must","{!lucene q.op=AND df=cat_s "+"cache="+random().nextBoolean()+"}A" );
+          }   
+        } else {
+          assertCatANot1(client, "filter", "{!lucene q.op=AND df=cat_s cache=false}A");
+        }
+        assertNull("no cache still",filterCache.get(catA));
+
+        if (random().nextBoolean()) {
+          if (random().nextBoolean()) {
+            assertCatANot1(client, "filter", "cat_s:A");
+          } else {
+            assertCatANot1(client, "filter");
+          }
+        } else {
+          assertCatANot1(client, "filter","{!lucene q.op=AND df=cat_s cache=true}A");
+        }
+        assertNotNull("got cached ",filterCache.get(catA));
+
+      } finally {
+        request.close();
+      }
+    }
+  }
+
+  private static void assertCatANot1(Client client, final String occur) throws Exception {
+    assertCatANot1(client, occur,  "{!lucene q.op=AND df=cat_s}A");
+  }
+
+  private static void assertCatANot1(Client client, final String occur, String catAclause) throws Exception {
+    client.testJQ( params("json","{ " +
+            " query : {" +
+            "  bool : {" +
+            "   " + occur + " : '"+ catAclause+ "'" +
+            "   must_not : '{!lucene v=\\'id:1\\'}'" +
+            "  }" +
+            " }" +
+            "}")
+        , "response/numFound==1"
+    );
+  }
+
   public static void doJsonRequestWithTag(Client client) throws Exception {
     addDocs(client);
 

solr/server/etc/jetty-ssl.xml

@@ -18,8 +18,8 @@
   <Set name="NeedClientAuth"><Property name="solr.jetty.ssl.needClientAuth" default="false"/></Set>
   <Set name="WantClientAuth"><Property name="solr.jetty.ssl.wantClientAuth" default="false"/></Set>
   <Set name="EndpointIdentificationAlgorithm"><Property name="solr.jetty.ssl.verifyClientHostName"/></Set>
-  <Set name="KeyStoreType"><Property name="solr.jetty.keystore.type" default="JKS"/></Set>
+  <Set name="KeyStoreType"><Property name="solr.jetty.keystore.type" default="PKCS12"/></Set>
-  <Set name="TrustStoreType"><Property name="solr.jetty.truststore.type" default="JKS"/></Set>
+  <Set name="TrustStoreType"><Property name="solr.jetty.truststore.type" default="PKCS12"/></Set>
 
   <!-- =========================================================== -->
   <!-- Create a TLS specific HttpConfiguration based on the        -->

solr/solr-ref-guide/src/enabling-ssl.adoc

@@ -36,25 +36,14 @@ In addition to `localhost` and `127.0.0.1`, this example includes a LAN IP addre
 
 [source,bash]
 ----
-keytool -genkeypair -alias solr-ssl -keyalg RSA -keysize 2048 -keypass secret -storepass secret -validity 9999 -keystore solr-ssl.keystore.jks -ext SAN=DNS:localhost,IP:192.168.1.3,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country"
+keytool -genkeypair -alias solr-ssl -keyalg RSA -keysize 2048 -keypass secret -storepass secret -validity 9999 -keystore solr-ssl.keystore.p12 -storetype PKCS12 -ext SAN=DNS:localhost,IP:192.168.1.3,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country"
 ----
 
-The above command will create a keystore file named `solr-ssl.keystore.jks` in the current directory.
+The above command will create a keystore file named `solr-ssl.keystore.p12` in the current directory.
 
 === Convert the Certificate and Key to PEM Format for Use with curl
 
-curl isn't capable of using JKS formatted keystores, so the JKS keystore needs to be converted to PEM format, which curl understands.
+Convert the PKCS12 format keystore, including both the certificate and the key, into PEM format using the http://www.openssl.org[`openssl`] command:
-
-First convert the JKS keystore into PKCS12 format using `keytool`:
-
-[source,bash]
-----
-keytool -importkeystore -srckeystore solr-ssl.keystore.jks -destkeystore solr-ssl.keystore.p12 -srcstoretype jks -deststoretype pkcs12
-----
-
-The keytool application will prompt you to create a destination keystore password and for the source keystore password, which was set when creating the keystore ("secret" in the example shown above).
-
-Next convert the PKCS12 format keystore, including both the certificate and the key, into PEM format using the http://www.openssl.org[`openssl`] command:
 
 [source,bash]
 ----
@@ -82,9 +71,9 @@ NOTE: If you setup Solr as a service on Linux using the steps outlined in <<taki
 SOLR_SSL_ENABLED=true
 # Uncomment to set SSL-related system properties
 # Be sure to update the paths to the correct keystore for your environment
-SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.jks
+SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.p12
 SOLR_SSL_KEY_STORE_PASSWORD=secret
-SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.jks
+SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.p12
 SOLR_SSL_TRUST_STORE_PASSWORD=secret
 # Require clients to authenticate
 SOLR_SSL_NEED_CLIENT_AUTH=false
@@ -95,9 +84,6 @@ SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false
 # SSL Certificates contain host/ip "peer name" information that is validated by default. Setting
 # this to false can be useful to disable these checks when re-using a certificate on many hosts
 SOLR_SSL_CHECK_PEER_NAME=true
-# Override Key/Trust Store types if necessary
-SOLR_SSL_KEY_STORE_TYPE=JKS
-SOLR_SSL_TRUST_STORE_TYPE=JKS
 ----
 
 When you start Solr, the `bin/solr` script includes the settings in `bin/solr.in.sh` and will pass these SSL-related system properties to the JVM.
@@ -115,9 +101,9 @@ REM to enable https module with custom jetty configuration.
 set SOLR_SSL_ENABLED=true
 REM Uncomment to set SSL-related system properties
 REM Be sure to update the paths to the correct keystore for your environment
-set SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.jks
+set SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.p12
 set SOLR_SSL_KEY_STORE_PASSWORD=secret
-set SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.jks
+set SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.p12
 set SOLR_SSL_TRUST_STORE_PASSWORD=secret
 REM Require clients to authenticate
 set SOLR_SSL_NEED_CLIENT_AUTH=false
@@ -128,9 +114,6 @@ set SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false
 REM SSL Certificates contain host/ip "peer name" information that is validated by default. Setting
 REM this to false can be useful to disable these checks when re-using a certificate on many hosts
 set SOLR_SSL_CHECK_PEER_NAME=true
-REM Override Key/Trust Store types if necessary
-set SOLR_SSL_KEY_STORE_TYPE=JKS
-set SOLR_SSL_TRUST_STORE_TYPE=JKS
 ----
 
 === Run Single Node Solr using SSL
@@ -403,7 +386,7 @@ Use `post.jar` to index some example documents to the SolrCloud collection creat
 ----
 cd example/exampledocs
 
-java -Djavax.net.ssl.keyStorePassword=secret -Djavax.net.ssl.keyStore=../../server/etc/solr-ssl.keystore.jks -Djavax.net.ssl.trustStore=../../server/etc/solr-ssl.keystore.jks -Djavax.net.ssl.trustStorePassword=secret -Durl=https://localhost:8984/solr/mycollection/update -jar post.jar *.xml
+java -Djavax.net.ssl.keyStorePassword=secret -Djavax.net.ssl.keyStore=../../server/etc/solr-ssl.keystore.p12 -Djavax.net.ssl.trustStore=../../server/etc/solr-ssl.keystore.p12 -Djavax.net.ssl.trustStorePassword=secret -Durl=https://localhost:8984/solr/mycollection/update -jar post.jar *.xml
 ----
 
 === Query Using curl
@@ -421,9 +404,9 @@ From a java client using SolrJ, index a document. In the code below, the `javax.
 
 [source,java]
 ----
-System.setProperty("javax.net.ssl.keyStore", "/path/to/solr-ssl.keystore.jks");
+System.setProperty("javax.net.ssl.keyStore", "/path/to/solr-ssl.keystore.p12");
 System.setProperty("javax.net.ssl.keyStorePassword", "secret");
-System.setProperty("javax.net.ssl.trustStore", "/path/to/solr-ssl.keystore.jks");
+System.setProperty("javax.net.ssl.trustStore", "/path/to/solr-ssl.keystore.p12");
 System.setProperty("javax.net.ssl.trustStorePassword", "secret");
 String zkHost = "127.0.0.1:2181";
 CloudSolrClient client = new CloudSolrClient.Builder().withZkHost(zkHost).build();

solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc

@@ -145,7 +145,7 @@ Then you can start the Exporter as follows (Linux).
 [source,bash]
 ----
 $ cd contrib/prometheus-exporter
-$ export JAVA_OPTS="-Djavax.net.ssl.trustStore=truststore.jks -Djavax.net.ssl.trustStorePassword=truststorePassword -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory -Dsolr.httpclient.config=basicauth.properties -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider -DzkDigestUsername=readonly-user -DzkDigestPassword=zkUserPassword"
+$ export JAVA_OPTS="-Djavax.net.ssl.trustStore=truststore.p12 -Djavax.net.ssl.trustStorePassword=truststorePassword -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory -Dsolr.httpclient.config=basicauth.properties -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider -DzkDigestUsername=readonly-user -DzkDigestPassword=zkUserPassword"
 $ export CLASSPATH_PREFIX="../../server/solr-webapp/webapp/WEB-INF/lib/commons-codec-1.11.jar"
 $ ./bin/solr-exporter -p 9854 -z zk1:2181,zk2:2181,zk3:2181 -f ./conf/solr-exporter-config.xml -n 16
 ----

solr/solr-ref-guide/src/other-parsers.adoc

@@ -188,7 +188,7 @@ A list of queries that *must not* appear in matching documents.
 A list of queries *should* appear in matching documents. For a BooleanQuery with no `must` queries, one or more `should` queries must match a document for the BooleanQuery to match.
 
 `filter`::
-A list of queries that *must* appear in matching documents. However, unlike `must`, the score of filter queries is ignored.
+A list of queries that *must* appear in matching documents. However, unlike `must`, the score of filter queries is ignored. Also, these queries are cached in filter cache. To avoid caching add either `cache=false` as local parameter, or `"cache":"false"` property to underneath Query DLS Object.  
 
 *Examples*