Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into gradle-master

This commit is contained in:
Dawid Weiss 2020-01-09 19:22:09 +01:00
parent 4599c51f0d 239d9a6726
commit d7726495c5
14 changed files with 178 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -123,6 +123,9 @@ Bug Fixes
* LUCENE-9084: Fix potential deadlock due to circular synchronization in AnalyzingInfixSuggester (Paul Ward)
* LUCENE-9115: NRTCachingDirectory no longer caches files of unknown size.
(Adrien Grand)
Other
---------------------

164
lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnumFrame.java
View File

@ -18,6 +18,7 @@ package org.apache.lucene.codecs.blocktree;
import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.index.IndexOptions;
@ -523,8 +524,7 @@ final class SegmentTermsEnumFrame {
assert prefixMatches(target);
// Loop over each entry (term or sub-block) in this block:
//nextTerm: while(nextEnt < entCount) {
nextTerm: while (true) {
do {
nextEnt++;
suffix = suffixesReader.readVInt();
@ -537,60 +537,37 @@ final class SegmentTermsEnumFrame {
// System.out.println(" cycle: term " + (nextEnt-1) + " (of " + entCount + ") suffix=" + brToString(suffixBytesRef));
// }
final int termLen = prefix + suffix;
startBytePos = suffixesReader.getPosition();
suffixesReader.skipBytes(suffix);
final int targetLimit = target.offset + (target.length < termLen ? target.length : termLen);
int targetPos = target.offset + prefix;
// Loop over bytes in the suffix, comparing to the target
final int cmp = Arrays.compareUnsigned(
suffixBytes, startBytePos, startBytePos + suffix,
target.bytes, target.offset + prefix, target.offset + target.length);
// Loop over bytes in the suffix, comparing to
// the target
int bytePos = startBytePos;
while(true) {
final int cmp;
final boolean stop;
if (targetPos < targetLimit) {
cmp = (suffixBytes[bytePos++]&0xFF) - (target.bytes[targetPos++]&0xFF);
stop = false;
} else {
assert targetPos == targetLimit;
cmp = termLen - target.length;
stop = true;
}
if (cmp < 0) {
// Current entry is still before the target;
// keep scanning
} else if (cmp > 0) {
// Done! Current entry is after target --
// return NOT_FOUND:
fillTerm();
if (cmp < 0) {
// Current entry is still before the target;
// keep scanning
//if (DEBUG) System.out.println(" not found");
return SeekStatus.NOT_FOUND;
} else {
// Exact match!
if (nextEnt == entCount) {
// We are done scanning this block
break nextTerm;
} else {
continue nextTerm;
}
} else if (cmp > 0) {
// This cannot be a sub-block because we
// would have followed the index to this
// sub-block from the start:
// Done! Current entry is after target --
// return NOT_FOUND:
fillTerm();
//if (DEBUG) System.out.println(" not found");
return SeekStatus.NOT_FOUND;
} else if (stop) {
// Exact match!
// This cannot be a sub-block because we
// would have followed the index to this
// sub-block from the start:
assert ste.termExists;
fillTerm();
//if (DEBUG) System.out.println(" found!");
return SeekStatus.FOUND;
}
assert ste.termExists;
fillTerm();
//if (DEBUG) System.out.println(" found!");
return SeekStatus.FOUND;
}
}
} while (nextEnt < entCount);
// It is possible (and OK) that terms index pointed us
// at this block, but, we scanned the entire block and
@ -631,7 +608,7 @@ final class SegmentTermsEnumFrame {
assert prefixMatches(target);
// Loop over each entry (term or sub-block) in this block:
nextTerm: while(nextEnt < entCount) {
while(nextEnt < entCount) {
nextEnt++;
@ -658,65 +635,48 @@ final class SegmentTermsEnumFrame {
lastSubFP = fp - subCode;
}
final int targetLimit = target.offset + (target.length < termLen ? target.length : termLen);
int targetPos = target.offset + prefix;
final int cmp = Arrays.compareUnsigned(
suffixBytes, startBytePos, startBytePos + suffix,
target.bytes, target.offset + prefix, target.offset + target.length);
// Loop over bytes in the suffix, comparing to
// the target
int bytePos = startBytePos;
while (true) {
final int cmp;
final boolean stop;
if (targetPos < targetLimit) {
cmp = (suffixBytes[bytePos++]&0xFF) - (target.bytes[targetPos++]&0xFF);
stop = false;
} else {
assert targetPos == targetLimit;
cmp = termLen - target.length;
stop = true;
}
if (cmp < 0) {
// Current entry is still before the target;
// keep scanning
} else if (cmp > 0) {
// Done! Current entry is after target --
// return NOT_FOUND:
fillTerm();
if (cmp < 0) {
// Current entry is still before the target;
// keep scanning
continue nextTerm;
} else if (cmp > 0) {
//if (DEBUG) System.out.println(" maybe done exactOnly=" + exactOnly + " ste.termExists=" + ste.termExists);
// Done! Current entry is after target --
// return NOT_FOUND:
fillTerm();
//if (DEBUG) System.out.println(" maybe done exactOnly=" + exactOnly + " ste.termExists=" + ste.termExists);
if (!exactOnly && !ste.termExists) {
//System.out.println(" now pushFrame");
// TODO this
// We are on a sub-block, and caller wants
// us to position to the next term after
// the target, so we must recurse into the
// sub-frame(s):
ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, termLen);
if (!exactOnly && !ste.termExists) {
//System.out.println(" now pushFrame");
// TODO this
// We are on a sub-block, and caller wants
// us to position to the next term after
// the target, so we must recurse into the
// sub-frame(s):
ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, termLen);
ste.currentFrame.loadBlock();
while (ste.currentFrame.next()) {
ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length());
ste.currentFrame.loadBlock();
while (ste.currentFrame.next()) {
ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length());
ste.currentFrame.loadBlock();
}
}
//if (DEBUG) System.out.println(" not found");
return SeekStatus.NOT_FOUND;
} else if (stop) {
// Exact match!
// This cannot be a sub-block because we
// would have followed the index to this
// sub-block from the start:
assert ste.termExists;
fillTerm();
//if (DEBUG) System.out.println(" found!");
return SeekStatus.FOUND;
}
//if (DEBUG) System.out.println(" not found");
return SeekStatus.NOT_FOUND;
} else {
// Exact match!
// This cannot be a sub-block because we
// would have followed the index to this
// sub-block from the start:
assert ste.termExists;
fillTerm();
//if (DEBUG) System.out.println(" found!");
return SeekStatus.FOUND;
}
}

2
lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
View File

@ -232,6 +232,8 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
bytes = context.mergeInfo.estimatedMergeBytes;
} else if (context.flushInfo != null) {
bytes = context.flushInfo.estimatedSegmentSize;
} else {
return false;
}
return (bytes <= maxMergeSizeBytes) && (bytes + cacheSize.get()) <= maxCachedBytes;

30
lucene/core/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java
View File

@ -136,4 +136,34 @@ public class TestNRTCachingDirectory extends BaseDirectoryTestCase {
nrtDir.close();
fsDir.close();
}
public void testUnknownFileSize() throws IOException {
Directory dir = newDirectory();
Directory nrtDir1 = new NRTCachingDirectory(dir, 1, 1) {
@Override
protected boolean doCacheWrite(String name, IOContext context) {
boolean cache = super.doCacheWrite(name, context);
assertTrue(cache);
return cache;
}
};
IOContext ioContext = new IOContext(new FlushInfo(3, 42));
nrtDir1.createOutput("foo", ioContext).close();
nrtDir1.createTempOutput("bar", "baz", ioContext).close();
Directory nrtDir2 = new NRTCachingDirectory(dir, 1, 1) {
@Override
protected boolean doCacheWrite(String name, IOContext context) {
boolean cache = super.doCacheWrite(name, context);
assertFalse(cache);
return cache;
}
};
ioContext = IOContext.DEFAULT;
nrtDir2.createOutput("foo", ioContext).close();
nrtDir2.createTempOutput("bar", "baz", ioContext).close();
dir.close();
}
}

21
solr/CHANGES.txt
View File

@ -210,6 +210,8 @@ Bug Fixes
* SOLR-13089: Fix lsof edge cases in the solr CLI script (Martijn Koster via janhoy)
* SOLR-14163: SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION needs to work with Jetty server/client SSL contexts (Kevin Risden)
Other Changes
---------------------
@ -224,6 +226,25 @@ Other Changes
* SOLR-14169: Fix 20 Resource Leak warnings in SolrJ's apache/solr/common (Andras Salamon via Tomás Fernández Löbbe)
================== 8.4.1 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
Versions of Major Components
---------------------
Apache Tika 1.19.1
Carrot2 3.16.0
Velocity 2.0 and Velocity Tools 3.0
Apache ZooKeeper 3.5.5
Jetty 9.4.19.v20190610
Bug Fixes
----------------------
* SOLR-14165: SolrResponse serialVersionUID has changed in a backward incompatible way (Andy Webb via noble)
================== 8.4.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

7
solr/bin/solr
View File

@ -206,14 +206,13 @@ if [ "$SOLR_SSL_ENABLED" == "true" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.truststore.type=$SOLR_SSL_TRUST_STORE_TYPE"
fi
if [ -n "$SOLR_SSL_NEED_CLIENT_AUTH" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.ssl.needClientAuth=$SOLR_SSL_NEED_CLIENT_AUTH"
fi
if [ -z "$SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION" ] ; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.ssl.verifyClientHostName=HTTPS"
fi
if [ -n "$SOLR_SSL_NEED_CLIENT_AUTH" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.ssl.needClientAuth=$SOLR_SSL_NEED_CLIENT_AUTH"
fi
if [ -n "$SOLR_SSL_WANT_CLIENT_AUTH" ]; then
SOLR_SSL_OPTS+=" -Dsolr.jetty.ssl.wantClientAuth=$SOLR_SSL_WANT_CLIENT_AUTH"
fi

4
solr/bin/solr.cmd
View File

@ -82,6 +82,10 @@ IF "%SOLR_SSL_ENABLED%"=="true" (
set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.jetty.truststore.type=%SOLR_SSL_TRUST_STORE_TYPE%"
)
IF NOT DEFINED SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION (
set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.jetty.ssl.verifyClientHostName=HTTPS"
)
IF DEFINED SOLR_SSL_NEED_CLIENT_AUTH (
set "SOLR_SSL_OPTS=!SOLR_SSL_OPTS! -Dsolr.jetty.ssl.needClientAuth=%SOLR_SSL_NEED_CLIENT_AUTH%"
)

1
solr/server/etc/jetty-ssl.xml
View File

@ -17,7 +17,6 @@
<Set name="TrustStorePassword"><Ref refid="trustStorePassword"/></Set>
<Set name="NeedClientAuth"><Property name="solr.jetty.ssl.needClientAuth" default="false"/></Set>
<Set name="WantClientAuth"><Property name="solr.jetty.ssl.wantClientAuth" default="false"/></Set>
<Set name="EndpointIdentificationAlgorithm"><Property name="solr.jetty.ssl.verifyClientHostName"/></Set>
<Set name="KeyStoreType"><Property name="solr.jetty.keystore.type" default="PKCS12"/></Set>
<Set name="TrustStoreType"><Property name="solr.jetty.truststore.type" default="PKCS12"/></Set>

10
solr/solr-ref-guide/src/enabling-ssl.adoc
View File

@ -66,7 +66,7 @@ NOTE: If you setup Solr as a service on Linux using the steps outlined in <<taki
.bin/solr.in.sh example SOLR_SSL_* configuration
[source,bash]
----
# Enables HTTPS. It is implictly true if you set SOLR_SSL_KEY_STORE. Use this config
# Enables HTTPS. It is implicitly true if you set SOLR_SSL_KEY_STORE. Use this config
# to enable https module with custom jetty configuration.
SOLR_SSL_ENABLED=true
# Uncomment to set SSL-related system properties
@ -79,8 +79,6 @@ SOLR_SSL_TRUST_STORE_PASSWORD=secret
SOLR_SSL_NEED_CLIENT_AUTH=false
# Enable clients to authenticate (but not require)
SOLR_SSL_WANT_CLIENT_AUTH=false
# Verify client's hostname during SSL handshake
SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false
# SSL Certificates contain host/ip "peer name" information that is validated by default. Setting
# this to false can be useful to disable these checks when re-using a certificate on many hosts
SOLR_SSL_CHECK_PEER_NAME=true
@ -89,14 +87,14 @@ SOLR_SSL_CHECK_PEER_NAME=true
When you start Solr, the `bin/solr` script includes the settings in `bin/solr.in.sh` and will pass these SSL-related system properties to the JVM.
.Client Authentication Settings
WARNING: Enable either SOLR_SSL_NEED_CLIENT_AUTH or SOLR_SSL_WANT_CLIENT_AUTH but not both at the same time. They are mutually exclusive and Jetty will select one of them which may not be what you expect. SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION should be set to true if you only want requests from authenticated host-names to be accepted.
WARNING: Enable either SOLR_SSL_NEED_CLIENT_AUTH or SOLR_SSL_WANT_CLIENT_AUTH but not both at the same time. They are mutually exclusive and Jetty will select one of them which may not be what you expect. SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION should be set to false if you want to disable hostname verification.
Similarly, when you start Solr on Windows, the `bin\solr.cmd` script includes the settings in `bin\solr.in.cmd` - uncomment and update the set of properties beginning with `SOLR_SSL_*` to pass these SSL-related system properties to the JVM:
.bin\solr.in.cmd example SOLR_SSL_* configuration
[source,text]
----
REM Enables HTTPS. It is implictly true if you set SOLR_SSL_KEY_STORE. Use this config
REM Enables HTTPS. It is implicitly true if you set SOLR_SSL_KEY_STORE. Use this config
REM to enable https module with custom jetty configuration.
set SOLR_SSL_ENABLED=true
REM Uncomment to set SSL-related system properties
@ -109,8 +107,6 @@ REM Require clients to authenticate
set SOLR_SSL_NEED_CLIENT_AUTH=false
REM Enable clients to authenticate (but not require)
set SOLR_SSL_WANT_CLIENT_AUTH=false
REM Verify client hostname during SSL handshake
set SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false
REM SSL Certificates contain host/ip "peer name" information that is validated by default. Setting
REM this to false can be useful to disable these checks when re-using a certificate on many hosts
set SOLR_SSL_CHECK_PEER_NAME=true

22
solr/solr-ref-guide/src/schema-api.adoc
View File

@ -73,7 +73,7 @@ bin/solr -e cloud -noprompt
== Modify the Schema
To add, remove or replace fields, dynamic field rules, copy field rules, or new field types, you can send a POST request to the `/collection/schema/` endpoint with a sequence of commands in JSON format to perform the requested actions. The following commands are supported:
To add, remove or replace fields, dynamic field rules, copy field rules, or new field types, you can send a POST request to the `/api/{collections|cores}/{name}/schema/` endpoint with a sequence of commands in JSON format to perform the requested actions. The following commands are supported:
* `add-field`: add a new field with parameters you provide.
* `delete-field`: delete a field.
@ -127,7 +127,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
"name":"sell_by",
"type":"pdate",
"stored":true }
}' http://localhost:8983/api/cores/gettingstarted/schema
}' http://localhost:8983/api/collections/gettingstarted/schema
----
====
--
@ -158,7 +158,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
----
curl -X POST -H 'Content-type:application/json' --data-binary '{
"delete-field" : { "name":"sell_by" }
}' http://localhost:8983/api/cores/gettingstarted/schema
}' http://localhost:8983/api/collections/gettingstarted/schema
----
====
--
@ -197,7 +197,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
"name":"sell_by",
"type":"date",
"stored":false }
}' http://localhost:8983/api/cores/gettingstarted/schema
}' http://localhost:8983/api/collections/gettingstarted/schema
----
====
--
@ -236,7 +236,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
"name":"*_s",
"type":"string",
"stored":true }
}' http://localhost:8983/api/cores/gettingstarted/schema
}' http://localhost:8983/api/collections/gettingstarted/schema
----
====
--
@ -267,7 +267,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
----
curl -X POST -H 'Content-type:application/json' --data-binary '{
"delete-dynamic-field":{ "name":"*_s" }
}' http://localhost:8983/api/cores/gettingstarted/schema
}' http://localhost:8983/api/collections/gettingstarted/schema
----
====
--
@ -388,7 +388,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
"queryAnalyzer":{
"tokenizer":{
"name":"keyword" }}}
}' http://localhost:8983/api/cores/gettingstarted/schema
}' http://localhost:8983/api/collections/gettingstarted/schema
----
====
--
@ -419,7 +419,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
----
curl -X POST -H 'Content-type:application/json' --data-binary '{
"delete-field-type":{ "name":"myNewTxtField" }
}' http://localhost:8983/api/cores/gettingstarted/schema
}' http://localhost:8983/api/collections/gettingstarted/schema
----
====
--
@ -464,7 +464,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
"analyzer":{
"tokenizer":{
"name":"standard" }}}
}' http://localhost:8983/api/cores/gettingstarted/schema
}' http://localhost:8983/api/collections/gettingstarted/schema
----
====
--
@ -510,7 +510,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
"add-copy-field":{
"source":"shelf",
"dest":[ "location", "catchall" ]}
}' http://localhost:8983/api/cores/gettingstarted/schema
}' http://localhost:8983/api/collections/gettingstarted/schema
----
====
--
@ -543,7 +543,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{
----
curl -X POST -H 'Content-type:application/json' --data-binary '{
"delete-copy-field":{ "source":"shelf", "dest":"location" }
}' http://localhost:8983/api/cores/gettingstarted/schema
}' http://localhost:8983/api/collections/gettingstarted/schema
----
====
--

3
solr/solrj/src/java/org/apache/solr/client/solrj/SolrResponse.java
View File

@ -37,6 +37,9 @@ import java.io.Serializable;
*/
public abstract class SolrResponse implements Serializable, MapWriter {
/** make this compatible with earlier versions */
private static final long serialVersionUID = -7931100103360242645L;
/** Elapsed time in milliseconds for the request as seen from the client. */
public abstract long getElapsedTime();

22
solr/solrj/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
View File

@ -136,7 +136,27 @@ public class SSLConfig {
}
public SslContextFactory.Client createClientContextFactory() {
return new SslContextFactory.Client();
if (! isSSLMode()) {
return null;
}
// else...
SslContextFactory.Client factory = new SslContextFactory.Client();
if (getKeyStore() != null) {
factory.setKeyStorePath(getKeyStore());
}
if (getKeyStorePassword() != null) {
factory.setKeyStorePassword(getKeyStorePassword());
}
if (isClientAuthMode()) {
if (getTrustStore() != null)
factory.setTrustStorePath(getTrustStore());
if (getTrustStorePassword() != null)
factory.setTrustStorePassword(getTrustStorePassword());
}
return factory;
}
private static SslContextFactory.Server configureSslFromSysProps() {

5
solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
View File

@ -919,7 +919,8 @@ public class Http2SolrClient extends SolrClient {
Http2SolrClient.defaultSSLConfig = null;
}
private static SslContextFactory.Client getDefaultSslContextFactory() {
/* package-private for testing */
static SslContextFactory.Client getDefaultSslContextFactory() {
String checkPeerNameStr = System.getProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME);
boolean sslCheckPeerName = true;
if (checkPeerNameStr == null || "false".equalsIgnoreCase(checkPeerNameStr)) {
@ -945,6 +946,8 @@ public class Http2SolrClient extends SolrClient {
(System.getProperty("javax.net.ssl.trustStorePassword"));
}
sslContextFactory.setEndpointIdentificationAlgorithm(System.getProperty("solr.jetty.ssl.verifyClientHostName"));
return sslContextFactory;
}
}

11
solr/solrj/src/test/org/apache/solr/client/solrj/impl/Http2SolrClientTest.java
View File

@ -45,6 +45,7 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.SuppressForbidden;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.junit.BeforeClass;
import org.junit.Test;
@ -588,6 +589,16 @@ public class Http2SolrClientTest extends SolrJettyTestBase {
}
}
@Test
public void testGetDefaultSslContextFactory() {
assertNull(Http2SolrClient.getDefaultSslContextFactory().getEndpointIdentificationAlgorithm());
System.setProperty("solr.jetty.ssl.verifyClientHostName", "HTTPS");
SslContextFactory.Client sslContextFactory = Http2SolrClient.getDefaultSslContextFactory();
assertEquals("HTTPS", sslContextFactory.getEndpointIdentificationAlgorithm());
System.clearProperty("solr.jetty.ssl.verifyClientHostName");
}
/**
* Missed tests :
* - set cookies via interceptor