Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into gradle-master

This commit is contained in:
Dawid Weiss 2019-12-30 09:24:46 +01:00
parent 11a946d145 ca6bd364fb
commit d79b678b39
15 changed files with 123 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
dev-tools/doap/lucene.rdf
View File

@ -67,6 +67,13 @@
</maintainer>
<!-- NOTE: please insert releases in numeric order, NOT chronologically. -->
<release>
<Version>
<name>lucene-8.4.0</name>
<created>2019-12-29</created>
<revision>8.4.0</revision>
</Version>
</release>
<release>
<Version>
<name>lucene-8.3.1</name>

7
dev-tools/doap/solr.rdf
View File

@ -67,6 +67,13 @@
</maintainer>
<!-- NOTE: please insert releases in numeric order, NOT chronologically. -->
<release>
<Version>
<name>solr-8.4.0</name>
<created>2019-12-29</created>
<revision>8.4.0</revision>
</Version>
</release>
<release>
<Version>
<name>solr-8.3.1</name>

7
lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
View File

@ -298,7 +298,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
"8.3.0-cfs",
"8.3.0-nocfs",
"8.3.1-cfs",
"8.3.1-nocfs"
"8.3.1-nocfs",
"8.4.0-cfs",
"8.4.0-nocfs"
};
public static String[] getOldNames() {
@ -311,7 +313,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
"sorted.8.1.1",
"sorted.8.2.0",
"sorted.8.3.0",
"sorted.8.3.1"
"sorted.8.3.1",
"sorted.8.4.0"
};
public static String[] getOldSortedNames() {

BIN
lucene/backward-codecs/src/test/org/apache/lucene/index/index.8.4.0-cfs.zip Normal file
View File

Binary file not shown.

BIN
lucene/backward-codecs/src/test/org/apache/lucene/index/index.8.4.0-nocfs.zip Normal file
View File

Binary file not shown.

BIN
lucene/backward-codecs/src/test/org/apache/lucene/index/sorted.8.4.0.zip Normal file
View File

Binary file not shown.

4
solr/CHANGES.txt
View File

@ -150,6 +150,8 @@ Upgrade Notes
If you prefer to keep the old (but insecure) serialization strategy, you can start your nodes using the
property: `-Dsolr.useUnsafeOverseerResponse=true`. Keep in mind that this will be removed in future version of Solr.
* SOLR-13808: add cache=false into uderneath BoolQParser's filter clause or {"bool":{"filter":..}} to avoid caching in
filterCache. (Mikhail Khludnev)
New Features
---------------------
@ -178,6 +180,8 @@ Improvements
* SOLR-13984: Java's SecurityManager sandbox can be enabled via environment variable,
SOLR_SECURITY_MANAGER_ENABLED=true. (rmuir)
* SOLR-13808: filter in BoolQParser and {"bool":{"filter":..}} in Query DSL are cached by default (Mikhail Khludnev)
Optimizations
---------------------
(No changes)

4
solr/bin/solr.in.cmd
View File

@ -141,8 +141,8 @@ REM SSL Certificates contain host/ip "peer name" information that is validated b
REM this to false can be useful to disable these checks when re-using a certificate on many hosts
REM set SOLR_SSL_CHECK_PEER_NAME=true
REM Override Key/Trust Store types if necessary
REM set SOLR_SSL_KEY_STORE_TYPE=JKS
REM set SOLR_SSL_TRUST_STORE_TYPE=JKS
REM set SOLR_SSL_KEY_STORE_TYPE=PKCS12
REM set SOLR_SSL_TRUST_STORE_TYPE=PKCS12
REM Uncomment if you want to override previously defined SSL values for HTTP client
REM otherwise keep them commented and the above values will automatically be set for HTTP clients

4
solr/bin/solr.in.sh
View File

@ -158,8 +158,8 @@
# this to false can be useful to disable these checks when re-using a certificate on many hosts
#SOLR_SSL_CHECK_PEER_NAME=true
# Override Key/Trust Store types if necessary
#SOLR_SSL_KEY_STORE_TYPE=JKS
#SOLR_SSL_TRUST_STORE_TYPE=JKS
#SOLR_SSL_KEY_STORE_TYPE=PKCS12
#SOLR_SSL_TRUST_STORE_TYPE=PKCS12
# Uncomment if you want to override previously defined SSL values for HTTP client
# otherwise keep them commented and the above values will automatically be set for HTTP clients

14
solr/core/src/java/org/apache/solr/search/BoolQParserPlugin.java
View File

@ -21,6 +21,7 @@ import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.query.FilterQuery;
import org.apache.solr.request.SolrQueryRequest;
/**
@ -49,7 +50,18 @@ public class BoolQParserPlugin extends QParserPlugin {
private void addQueries(BooleanQuery.Builder builder, String[] subQueries, BooleanClause.Occur occur) throws SyntaxError {
if (subQueries != null) {
for (String subQuery : subQueries) {
builder.add(subQuery(subQuery, null).parse(), occur);
final QParser subParser = subQuery(subQuery, null);
Query extQuery;
if (BooleanClause.Occur.FILTER.equals(occur)) {
extQuery = subParser.getQuery();
if (!(extQuery instanceof ExtendedQuery) || (
((ExtendedQuery) extQuery).getCache())) {
extQuery = new FilterQuery(extQuery);
}
} else {
extQuery = subParser.parse();
}
builder.add(extQuery, occur);
}
}
}

80
solr/core/src/test/org/apache/solr/search/json/TestJsonRequest.java
View File

@ -16,11 +16,17 @@
*/
package org.apache.solr.search.json;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.JSONTestUtil;
import org.apache.solr.SolrTestCaseHS;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.CaffeineCache;
import org.apache.solr.search.DocSet;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@ -31,6 +37,7 @@ public class TestJsonRequest extends SolrTestCaseHS {
private static SolrInstances servers; // for distributed testing
@SuppressWarnings("deprecation")
@BeforeClass
public static void beforeTests() throws Exception {
systemSetPropertySolrDisableShardsWhitelist("true");
@ -44,6 +51,7 @@ public class TestJsonRequest extends SolrTestCaseHS {
}
}
@SuppressWarnings("deprecation")
@AfterClass
public static void afterTests() throws Exception {
JSONTestUtil.failRepeatedKeys = false;
@ -303,17 +311,9 @@ public class TestJsonRequest extends SolrTestCaseHS {
, "response/numFound==1"
);
client.testJQ( params("json","{ " +
" query : {" +
" bool : {" +
" must : '{!lucene q.op=AND df=cat_s}A'" +
" must_not : '{!lucene v=\\'id:1\\'}'" +
" }" +
" }" +
"}")
, "response/numFound==1"
);
assertCatANot1(client, "must");
testFilterCachingLocally(client);
client.testJQ( params("json","{" +
" query : '*:*'," +
@ -407,6 +407,64 @@ public class TestJsonRequest extends SolrTestCaseHS {
}
private static void testFilterCachingLocally(Client client) throws Exception {
if(client.getClientProvider()==null) {
final SolrQueryRequest request = req();
try {
final CaffeineCache<Query,DocSet> filterCache = (CaffeineCache<Query,DocSet>) request.getSearcher().getFilterCache();
filterCache.clear();
final TermQuery catA = new TermQuery(new Term("cat_s", "A"));
assertNull("cache is empty",filterCache.get(catA));
if(random().nextBoolean()) {
if(random().nextBoolean()) {
if(random().nextBoolean()) {
assertCatANot1(client, "must");
}else {
assertCatANot1(client, "must", "cat_s:A");
}
} else {
assertCatANot1(client, "must","{!lucene q.op=AND df=cat_s "+"cache="+random().nextBoolean()+"}A" );
}
} else {
assertCatANot1(client, "filter", "{!lucene q.op=AND df=cat_s cache=false}A");
}
assertNull("no cache still",filterCache.get(catA));
if (random().nextBoolean()) {
if (random().nextBoolean()) {
assertCatANot1(client, "filter", "cat_s:A");
} else {
assertCatANot1(client, "filter");
}
} else {
assertCatANot1(client, "filter","{!lucene q.op=AND df=cat_s cache=true}A");
}
assertNotNull("got cached ",filterCache.get(catA));
} finally {
request.close();
}
}
}
private static void assertCatANot1(Client client, final String occur) throws Exception {
assertCatANot1(client, occur, "{!lucene q.op=AND df=cat_s}A");
}
private static void assertCatANot1(Client client, final String occur, String catAclause) throws Exception {
client.testJQ( params("json","{ " +
" query : {" +
" bool : {" +
" " + occur + " : '"+ catAclause+ "'" +
" must_not : '{!lucene v=\\'id:1\\'}'" +
" }" +
" }" +
"}")
, "response/numFound==1"
);
}
public static void doJsonRequestWithTag(Client client) throws Exception {
addDocs(client);

4
solr/server/etc/jetty-ssl.xml
View File

@ -18,8 +18,8 @@
<Set name="NeedClientAuth"><Property name="solr.jetty.ssl.needClientAuth" default="false"/></Set>
<Set name="WantClientAuth"><Property name="solr.jetty.ssl.wantClientAuth" default="false"/></Set>
<Set name="EndpointIdentificationAlgorithm"><Property name="solr.jetty.ssl.verifyClientHostName"/></Set>
<Set name="KeyStoreType"><Property name="solr.jetty.keystore.type" default="JKS"/></Set>
<Set name="TrustStoreType"><Property name="solr.jetty.truststore.type" default="JKS"/></Set>
<Set name="KeyStoreType"><Property name="solr.jetty.keystore.type" default="PKCS12"/></Set>
<Set name="TrustStoreType"><Property name="solr.jetty.truststore.type" default="PKCS12"/></Set>
<!-- =========================================================== -->
<!-- Create a TLS specific HttpConfiguration based on the -->

37
solr/solr-ref-guide/src/enabling-ssl.adoc
View File

@ -36,25 +36,14 @@ In addition to `localhost` and `127.0.0.1`, this example includes a LAN IP addre
[source,bash]
----
keytool -genkeypair -alias solr-ssl -keyalg RSA -keysize 2048 -keypass secret -storepass secret -validity 9999 -keystore solr-ssl.keystore.jks -ext SAN=DNS:localhost,IP:192.168.1.3,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country"
keytool -genkeypair -alias solr-ssl -keyalg RSA -keysize 2048 -keypass secret -storepass secret -validity 9999 -keystore solr-ssl.keystore.p12 -storetype PKCS12 -ext SAN=DNS:localhost,IP:192.168.1.3,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country"
----
The above command will create a keystore file named `solr-ssl.keystore.jks` in the current directory.
The above command will create a keystore file named `solr-ssl.keystore.p12` in the current directory.
=== Convert the Certificate and Key to PEM Format for Use with curl
curl isn't capable of using JKS formatted keystores, so the JKS keystore needs to be converted to PEM format, which curl understands.
First convert the JKS keystore into PKCS12 format using `keytool`:
[source,bash]
----
keytool -importkeystore -srckeystore solr-ssl.keystore.jks -destkeystore solr-ssl.keystore.p12 -srcstoretype jks -deststoretype pkcs12
----
The keytool application will prompt you to create a destination keystore password and for the source keystore password, which was set when creating the keystore ("secret" in the example shown above).
Next convert the PKCS12 format keystore, including both the certificate and the key, into PEM format using the http://www.openssl.org[`openssl`] command:
Convert the PKCS12 format keystore, including both the certificate and the key, into PEM format using the http://www.openssl.org[`openssl`] command:
[source,bash]
----
@ -82,9 +71,9 @@ NOTE: If you setup Solr as a service on Linux using the steps outlined in <<taki
SOLR_SSL_ENABLED=true
# Uncomment to set SSL-related system properties
# Be sure to update the paths to the correct keystore for your environment
SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.jks
SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.p12
SOLR_SSL_KEY_STORE_PASSWORD=secret
SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.jks
SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.p12
SOLR_SSL_TRUST_STORE_PASSWORD=secret
# Require clients to authenticate
SOLR_SSL_NEED_CLIENT_AUTH=false
@ -95,9 +84,6 @@ SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false
# SSL Certificates contain host/ip "peer name" information that is validated by default. Setting
# this to false can be useful to disable these checks when re-using a certificate on many hosts
SOLR_SSL_CHECK_PEER_NAME=true
# Override Key/Trust Store types if necessary
SOLR_SSL_KEY_STORE_TYPE=JKS
SOLR_SSL_TRUST_STORE_TYPE=JKS
----
When you start Solr, the `bin/solr` script includes the settings in `bin/solr.in.sh` and will pass these SSL-related system properties to the JVM.
@ -115,9 +101,9 @@ REM to enable https module with custom jetty configuration.
set SOLR_SSL_ENABLED=true
REM Uncomment to set SSL-related system properties
REM Be sure to update the paths to the correct keystore for your environment
set SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.jks
set SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.p12
set SOLR_SSL_KEY_STORE_PASSWORD=secret
set SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.jks
set SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.p12
set SOLR_SSL_TRUST_STORE_PASSWORD=secret
REM Require clients to authenticate
set SOLR_SSL_NEED_CLIENT_AUTH=false
@ -128,9 +114,6 @@ set SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false
REM SSL Certificates contain host/ip "peer name" information that is validated by default. Setting
REM this to false can be useful to disable these checks when re-using a certificate on many hosts
set SOLR_SSL_CHECK_PEER_NAME=true
REM Override Key/Trust Store types if necessary
set SOLR_SSL_KEY_STORE_TYPE=JKS
set SOLR_SSL_TRUST_STORE_TYPE=JKS
----
=== Run Single Node Solr using SSL
@ -403,7 +386,7 @@ Use `post.jar` to index some example documents to the SolrCloud collection creat
----
cd example/exampledocs
java -Djavax.net.ssl.keyStorePassword=secret -Djavax.net.ssl.keyStore=../../server/etc/solr-ssl.keystore.jks -Djavax.net.ssl.trustStore=../../server/etc/solr-ssl.keystore.jks -Djavax.net.ssl.trustStorePassword=secret -Durl=https://localhost:8984/solr/mycollection/update -jar post.jar *.xml
java -Djavax.net.ssl.keyStorePassword=secret -Djavax.net.ssl.keyStore=../../server/etc/solr-ssl.keystore.p12 -Djavax.net.ssl.trustStore=../../server/etc/solr-ssl.keystore.p12 -Djavax.net.ssl.trustStorePassword=secret -Durl=https://localhost:8984/solr/mycollection/update -jar post.jar *.xml
----
=== Query Using curl
@ -421,9 +404,9 @@ From a java client using SolrJ, index a document. In the code below, the `javax.
[source,java]
----
System.setProperty("javax.net.ssl.keyStore", "/path/to/solr-ssl.keystore.jks");
System.setProperty("javax.net.ssl.keyStore", "/path/to/solr-ssl.keystore.p12");
System.setProperty("javax.net.ssl.keyStorePassword", "secret");
System.setProperty("javax.net.ssl.trustStore", "/path/to/solr-ssl.keystore.jks");
System.setProperty("javax.net.ssl.trustStore", "/path/to/solr-ssl.keystore.p12");
System.setProperty("javax.net.ssl.trustStorePassword", "secret");
String zkHost = "127.0.0.1:2181";
CloudSolrClient client = new CloudSolrClient.Builder().withZkHost(zkHost).build();

2
solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc
View File

@ -145,7 +145,7 @@ Then you can start the Exporter as follows (Linux).
[source,bash]
----
$ cd contrib/prometheus-exporter
$ export JAVA_OPTS="-Djavax.net.ssl.trustStore=truststore.jks -Djavax.net.ssl.trustStorePassword=truststorePassword -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory -Dsolr.httpclient.config=basicauth.properties -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider -DzkDigestUsername=readonly-user -DzkDigestPassword=zkUserPassword"
$ export JAVA_OPTS="-Djavax.net.ssl.trustStore=truststore.p12 -Djavax.net.ssl.trustStorePassword=truststorePassword -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory -Dsolr.httpclient.config=basicauth.properties -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider -DzkDigestUsername=readonly-user -DzkDigestPassword=zkUserPassword"
$ export CLASSPATH_PREFIX="../../server/solr-webapp/webapp/WEB-INF/lib/commons-codec-1.11.jar"
$ ./bin/solr-exporter -p 9854 -z zk1:2181,zk2:2181,zk3:2181 -f ./conf/solr-exporter-config.xml -n 16
----

2
solr/solr-ref-guide/src/other-parsers.adoc
View File

@ -188,7 +188,7 @@ A list of queries that *must not* appear in matching documents.
A list of queries *should* appear in matching documents. For a BooleanQuery with no `must` queries, one or more `should` queries must match a document for the BooleanQuery to match.
`filter`::
A list of queries that *must* appear in matching documents. However, unlike `must`, the score of filter queries is ignored.
A list of queries that *must* appear in matching documents. However, unlike `must`, the score of filter queries is ignored. Also, these queries are cached in filter cache. To avoid caching add either `cache=false` as local parameter, or `"cache":"false"` property to underneath Query DLS Object.
*Examples*