Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-14186: Enforce CRLF in Windows files with .gitattributes (#1163)

This commit is contained in:
Jason Gerlowski 2020-01-16 08:30:39 -05:00 committed by GitHub
parent 7ea7ed72ac
commit 424ace6f5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 3137 additions and 3092 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
solr/.gitattributes vendored Normal file
View File

@ -0,0 +1,45 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Handles all files not treated particularly below
* text=auto
# -nix specific
bin/solr text eol=lf
bin/init.d/solr text eol=lf
bin/post text eol=lf
*.bash text eol=lf
*.sh text eol=lf
# Windows specific
*.bat text eol=crlf
*.cmd text eol=crlf
*.ps1 text eol=crlf
# Graphics - no eol normalization
*.png binary
*.jpg binary
*.jpeg binary
*.gif binary
*.tif binary
*.tiff binary
*.ico binary
*.svg binary
*.eps binary
# Other files to ignore from eol normalization
licenses/* -text
solr-ref-guide/src/fonts/Inconsolata/OFL.txt -text

4158
solr/bin/solr.cmd
View File

File diff suppressed because it is too large Load Diff

400
solr/bin/solr.in.cmd
View File

@ -1,200 +1,200 @@
@REM
@REM Licensed to the Apache Software Foundation (ASF) under one or more
@REM contributor license agreements. See the NOTICE file distributed with
@REM this work for additional information regarding copyright ownership.
@REM The ASF licenses this file to You under the Apache License, Version 2.0
@REM (the "License"); you may not use this file except in compliance with
@REM the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing, software
@REM distributed under the License is distributed on an "AS IS" BASIS,
@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@REM See the License for the specific language governing permissions and
@REM limitations under the License.
@echo off
REM Settings here will override settings in existing env vars or in bin/solr. The default shipped state
REM of this file is completely commented.
REM By default the script will use JAVA_HOME to determine which java
REM to use, but you can set a specific path for Solr to use without
REM affecting other Java applications on your server/workstation.
REM set SOLR_JAVA_HOME=
REM Increase Java Min/Max Heap as needed to support your indexing / query needs
REM set SOLR_JAVA_MEM=-Xms512m -Xmx512m
REM Configure verbose GC logging:
REM For Java 8: if this is set, additional params will be added to specify the log file & rotation
REM For Java 9 or higher: GC_LOG_OPTS is currently not supported. If you set it, the startup script will exit with failure.
REM set GC_LOG_OPTS=-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime
REM Various GC settings have shown to work well for a number of common Solr workloads.
REM See solr.cmd GC_TUNE for the default list.
REM set GC_TUNE=-XX:SurvivorRatio=4
REM set GC_TUNE=%GC_TUNE% -XX:TargetSurvivorRatio=90
REM set GC_TUNE=%GC_TUNE% -XX:MaxTenuringThreshold=8
REM set GC_TUNE=%GC_TUNE% -XX:+UseConcMarkSweepGC
REM set GC_TUNE=%GC_TUNE% -XX:ConcGCThreads=4
REM set GC_TUNE=%GC_TUNE% -XX:ParallelGCThreads=4
REM set GC_TUNE=%GC_TUNE% -XX:+CMSScavengeBeforeRemark
REM set GC_TUNE=%GC_TUNE% -XX:PretenureSizeThreshold=64m
REM set GC_TUNE=%GC_TUNE% -XX:+UseCMSInitiatingOccupancyOnly
REM set GC_TUNE=%GC_TUNE% -XX:CMSInitiatingOccupancyFraction=50
REM set GC_TUNE=%GC_TUNE% -XX:CMSMaxAbortablePrecleanTime=6000
REM set GC_TUNE=%GC_TUNE% -XX:+CMSParallelRemarkEnabled
REM set GC_TUNE=%GC_TUNE% -XX:+ParallelRefProcEnabled
REM set GC_TUNE=%GC_TUNE% -XX:-OmitStackTraceInFastThrow etc.
REM Set the ZooKeeper connection string if using an external ZooKeeper ensemble
REM e.g. host1:2181,host2:2181/chroot
REM Leave empty if not using SolrCloud
REM set ZK_HOST=
REM Set the ZooKeeper client timeout (for SolrCloud mode)
REM set ZK_CLIENT_TIMEOUT=15000
REM By default the start script uses "localhost"; override the hostname here
REM for production SolrCloud environments to control the hostname exposed to cluster state
REM set SOLR_HOST=192.168.1.1
REM By default Solr will try to connect to Zookeeper with 30 seconds in timeout; override the timeout if needed
REM set SOLR_WAIT_FOR_ZK=30
REM By default the start script uses UTC; override the timezone if needed
REM set SOLR_TIMEZONE=UTC
REM Set to true to activate the JMX RMI connector to allow remote JMX client applications
REM to monitor the JVM hosting Solr; set to "false" to disable that behavior
REM (false is recommended in production environments)
REM set ENABLE_REMOTE_JMX_OPTS=false
REM The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here
REM set RMI_PORT=18983
REM Anything you add to the SOLR_OPTS variable will be included in the java
REM start command line as-is, in ADDITION to other options. If you specify the
REM -a option on start script, those options will be appended as well. Examples:
REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.autoSoftCommit.maxTime=3000
REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.autoCommit.maxTime=60000
REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.clustering.enabled=true
REM Path to a directory for Solr to store cores and their data. By default, Solr will use server\solr
REM If solr.xml is not stored in ZooKeeper, this directory needs to contain solr.xml
REM set SOLR_HOME=
REM Path to a directory that Solr will use as root for data folders for each core.
REM If not set, defaults to <instance_dir>/data. Overridable per core through 'dataDir' core property
REM set SOLR_DATA_HOME=
REM Changes the logging level. Valid values: ALL, TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF. Default is INFO
REM This is an alternative to changing the rootLogger in log4j2.xml
REM set SOLR_LOG_LEVEL=INFO
REM Location where Solr should write logs to. Absolute or relative to solr start dir
REM set SOLR_LOGS_DIR=logs
REM Enables log rotation before starting Solr. Setting SOLR_LOG_PRESTART_ROTATION=true will let Solr take care of pre
REM start rotation of logs. This is false by default as log4j2 handles this for us. If you choose to use another log
REM framework that cannot do startup rotation, you may want to enable this to let Solr rotate logs on startup.
REM set SOLR_LOG_PRESTART_ROTATION=false
REM Enables jetty request log for all requests
REM set SOLR_REQUESTLOG_ENABLED=false
REM Sets the port Solr binds to, default is 8983
REM set SOLR_PORT=8983
REM Sets the network interface the Solr binds to. To prevent administrators from
REM accidentally exposing Solr more widely than intended, this defaults to 127.0.0.1.
REM Administrators should think carefully about their deployment environment and
REM set this value as narrowly as required before going to production. In
REM environments where security is not a concern, 0.0.0.0 can be used to allow
REM Solr to accept connections on all network interfaces.
REM set SOLR_JETTY_HOST=127.0.0.1
REM Restrict access to solr by IP address.
REM Specify a comma-separated list of addresses or networks, for example:
REM 127.0.0.1, 192.168.0.0/24, [::1], [2000:123:4:5::]/64
REM set SOLR_IP_WHITELIST=
REM Block access to solr from specific IP addresses.
REM Specify a comma-separated list of addresses or networks, for example:
REM 127.0.0.1, 192.168.0.0/24, [::1], [2000:123:4:5::]/64
REM set SOLR_IP_BLACKLIST=
REM Enables HTTPS. It is implictly true if you set SOLR_SSL_KEY_STORE. Use this config
REM to enable https module with custom jetty configuration.
REM set SOLR_SSL_ENABLED=true
REM Uncomment to set SSL-related system properties
REM Be sure to update the paths to the correct keystore for your environment
REM set SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.jks
REM set SOLR_SSL_KEY_STORE_PASSWORD=secret
REM set SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.jks
REM set SOLR_SSL_TRUST_STORE_PASSWORD=secret
REM Require clients to authenticate
REM set SOLR_SSL_NEED_CLIENT_AUTH=false
REM Enable clients to authenticate (but not require)
REM set SOLR_SSL_WANT_CLIENT_AUTH=false
REM Verify client hostname during SSL handshake
REM set SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false
REM SSL Certificates contain host/ip "peer name" information that is validated by default. Setting
REM this to false can be useful to disable these checks when re-using a certificate on many hosts
REM set SOLR_SSL_CHECK_PEER_NAME=true
REM Override Key/Trust Store types if necessary
REM set SOLR_SSL_KEY_STORE_TYPE=PKCS12
REM set SOLR_SSL_TRUST_STORE_TYPE=PKCS12
REM Uncomment if you want to override previously defined SSL values for HTTP client
REM otherwise keep them commented and the above values will automatically be set for HTTP clients
REM set SOLR_SSL_CLIENT_KEY_STORE=
REM set SOLR_SSL_CLIENT_KEY_STORE_PASSWORD=
REM set SOLR_SSL_CLIENT_TRUST_STORE=
REM set SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=
REM set SOLR_SSL_CLIENT_KEY_STORE_TYPE=
REM set SOLR_SSL_CLIENT_TRUST_STORE_TYPE=
REM Sets path of Hadoop credential provider (hadoop.security.credential.provider.path property) and
REM enables usage of credential store.
REM Credential provider should store the following keys:
REM * solr.jetty.keystore.password
REM * solr.jetty.truststore.password
REM Set the two below if you want to set specific store passwords for HTTP client
REM * javax.net.ssl.keyStorePassword
REM * javax.net.ssl.trustStorePassword
REM More info: https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html
REM set SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=localjceks://file/home/solr/hadoop-credential-provider.jceks
REM set SOLR_OPTS=" -Dsolr.ssl.credential.provider.chain=hadoop"
REM Settings for authentication
REM Please configure only one of SOLR_AUTHENTICATION_CLIENT_BUILDER or SOLR_AUTH_TYPE parameters
REM set SOLR_AUTHENTICATION_CLIENT_BUILDER=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory
REM set SOLR_AUTH_TYPE=basic
REM set SOLR_AUTHENTICATION_OPTS="-Dbasicauth=solr:SolrRocks"
REM Settings for ZK ACL
REM set SOLR_ZK_CREDS_AND_ACLS=-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider ^
REM -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider ^
REM -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD ^
REM -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD
REM set SOLR_OPTS=%SOLR_OPTS% %SOLR_ZK_CREDS_AND_ACLS%
REM When running Solr in non-cloud mode and if planning to do distributed search (using the "shards" parameter), the
REM list of hosts needs to be whitelisted or Solr will forbid the request. The whitelist can be configured in solr.xml,
REM or if you are using the OOTB solr.xml, can be specified using the system property "solr.shardsWhitelist". Alternatively
REM host checking can be disabled by using the system property "solr.disable.shardsWhitelist"
REM set SOLR_OPTS="%SOLR_OPTS% -Dsolr.shardsWhitelist=http://localhost:8983,http://localhost:8984"
REM For a visual indication in the Admin UI of what type of environment this cluster is, configure
REM a -Dsolr.environment property below. Valid values are prod, stage, test, dev, with an optional
REM label or color, e.g. -Dsolr.environment=test,label=Functional+test,color=brown
REM SOLR_OPTS="$SOLR_OPTS -Dsolr.environment=prod"
REM Runs solr in a java security manager sandbox. This can protect against some attacks.
REM Runtime properties are passed to the security policy file (server\etc\security.policy)
REM You can also tweak via standard JDK files such as ~\.java.policy, see https://s.apache.org/java8policy
REM This is experimental! It may not work at all with Hadoop/HDFS features.
REM set SOLR_SECURITY_MANAGER_ENABLED=false
@REM
@REM Licensed to the Apache Software Foundation (ASF) under one or more
@REM contributor license agreements. See the NOTICE file distributed with
@REM this work for additional information regarding copyright ownership.
@REM The ASF licenses this file to You under the Apache License, Version 2.0
@REM (the "License"); you may not use this file except in compliance with
@REM the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing, software
@REM distributed under the License is distributed on an "AS IS" BASIS,
@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@REM See the License for the specific language governing permissions and
@REM limitations under the License.
@echo off
REM Settings here will override settings in existing env vars or in bin/solr. The default shipped state
REM of this file is completely commented.
REM By default the script will use JAVA_HOME to determine which java
REM to use, but you can set a specific path for Solr to use without
REM affecting other Java applications on your server/workstation.
REM set SOLR_JAVA_HOME=
REM Increase Java Min/Max Heap as needed to support your indexing / query needs
REM set SOLR_JAVA_MEM=-Xms512m -Xmx512m
REM Configure verbose GC logging:
REM For Java 8: if this is set, additional params will be added to specify the log file & rotation
REM For Java 9 or higher: GC_LOG_OPTS is currently not supported. If you set it, the startup script will exit with failure.
REM set GC_LOG_OPTS=-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime
REM Various GC settings have shown to work well for a number of common Solr workloads.
REM See solr.cmd GC_TUNE for the default list.
REM set GC_TUNE=-XX:SurvivorRatio=4
REM set GC_TUNE=%GC_TUNE% -XX:TargetSurvivorRatio=90
REM set GC_TUNE=%GC_TUNE% -XX:MaxTenuringThreshold=8
REM set GC_TUNE=%GC_TUNE% -XX:+UseConcMarkSweepGC
REM set GC_TUNE=%GC_TUNE% -XX:ConcGCThreads=4
REM set GC_TUNE=%GC_TUNE% -XX:ParallelGCThreads=4
REM set GC_TUNE=%GC_TUNE% -XX:+CMSScavengeBeforeRemark
REM set GC_TUNE=%GC_TUNE% -XX:PretenureSizeThreshold=64m
REM set GC_TUNE=%GC_TUNE% -XX:+UseCMSInitiatingOccupancyOnly
REM set GC_TUNE=%GC_TUNE% -XX:CMSInitiatingOccupancyFraction=50
REM set GC_TUNE=%GC_TUNE% -XX:CMSMaxAbortablePrecleanTime=6000
REM set GC_TUNE=%GC_TUNE% -XX:+CMSParallelRemarkEnabled
REM set GC_TUNE=%GC_TUNE% -XX:+ParallelRefProcEnabled
REM set GC_TUNE=%GC_TUNE% -XX:-OmitStackTraceInFastThrow etc.
REM Set the ZooKeeper connection string if using an external ZooKeeper ensemble
REM e.g. host1:2181,host2:2181/chroot
REM Leave empty if not using SolrCloud
REM set ZK_HOST=
REM Set the ZooKeeper client timeout (for SolrCloud mode)
REM set ZK_CLIENT_TIMEOUT=15000
REM By default the start script uses "localhost"; override the hostname here
REM for production SolrCloud environments to control the hostname exposed to cluster state
REM set SOLR_HOST=192.168.1.1
REM By default Solr will try to connect to Zookeeper with 30 seconds in timeout; override the timeout if needed
REM set SOLR_WAIT_FOR_ZK=30
REM By default the start script uses UTC; override the timezone if needed
REM set SOLR_TIMEZONE=UTC
REM Set to true to activate the JMX RMI connector to allow remote JMX client applications
REM to monitor the JVM hosting Solr; set to "false" to disable that behavior
REM (false is recommended in production environments)
REM set ENABLE_REMOTE_JMX_OPTS=false
REM The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here
REM set RMI_PORT=18983
REM Anything you add to the SOLR_OPTS variable will be included in the java
REM start command line as-is, in ADDITION to other options. If you specify the
REM -a option on start script, those options will be appended as well. Examples:
REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.autoSoftCommit.maxTime=3000
REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.autoCommit.maxTime=60000
REM set SOLR_OPTS=%SOLR_OPTS% -Dsolr.clustering.enabled=true
REM Path to a directory for Solr to store cores and their data. By default, Solr will use server\solr
REM If solr.xml is not stored in ZooKeeper, this directory needs to contain solr.xml
REM set SOLR_HOME=
REM Path to a directory that Solr will use as root for data folders for each core.
REM If not set, defaults to <instance_dir>/data. Overridable per core through 'dataDir' core property
REM set SOLR_DATA_HOME=
REM Changes the logging level. Valid values: ALL, TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF. Default is INFO
REM This is an alternative to changing the rootLogger in log4j2.xml
REM set SOLR_LOG_LEVEL=INFO
REM Location where Solr should write logs to. Absolute or relative to solr start dir
REM set SOLR_LOGS_DIR=logs
REM Enables log rotation before starting Solr. Setting SOLR_LOG_PRESTART_ROTATION=true will let Solr take care of pre
REM start rotation of logs. This is false by default as log4j2 handles this for us. If you choose to use another log
REM framework that cannot do startup rotation, you may want to enable this to let Solr rotate logs on startup.
REM set SOLR_LOG_PRESTART_ROTATION=false
REM Enables jetty request log for all requests
REM set SOLR_REQUESTLOG_ENABLED=false
REM Sets the port Solr binds to, default is 8983
REM set SOLR_PORT=8983
REM Sets the network interface the Solr binds to. To prevent administrators from
REM accidentally exposing Solr more widely than intended, this defaults to 127.0.0.1.
REM Administrators should think carefully about their deployment environment and
REM set this value as narrowly as required before going to production. In
REM environments where security is not a concern, 0.0.0.0 can be used to allow
REM Solr to accept connections on all network interfaces.
REM set SOLR_JETTY_HOST=127.0.0.1
REM Restrict access to solr by IP address.
REM Specify a comma-separated list of addresses or networks, for example:
REM 127.0.0.1, 192.168.0.0/24, [::1], [2000:123:4:5::]/64
REM set SOLR_IP_WHITELIST=
REM Block access to solr from specific IP addresses.
REM Specify a comma-separated list of addresses or networks, for example:
REM 127.0.0.1, 192.168.0.0/24, [::1], [2000:123:4:5::]/64
REM set SOLR_IP_BLACKLIST=
REM Enables HTTPS. It is implictly true if you set SOLR_SSL_KEY_STORE. Use this config
REM to enable https module with custom jetty configuration.
REM set SOLR_SSL_ENABLED=true
REM Uncomment to set SSL-related system properties
REM Be sure to update the paths to the correct keystore for your environment
REM set SOLR_SSL_KEY_STORE=etc/solr-ssl.keystore.jks
REM set SOLR_SSL_KEY_STORE_PASSWORD=secret
REM set SOLR_SSL_TRUST_STORE=etc/solr-ssl.keystore.jks
REM set SOLR_SSL_TRUST_STORE_PASSWORD=secret
REM Require clients to authenticate
REM set SOLR_SSL_NEED_CLIENT_AUTH=false
REM Enable clients to authenticate (but not require)
REM set SOLR_SSL_WANT_CLIENT_AUTH=false
REM Verify client hostname during SSL handshake
REM set SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION=false
REM SSL Certificates contain host/ip "peer name" information that is validated by default. Setting
REM this to false can be useful to disable these checks when re-using a certificate on many hosts
REM set SOLR_SSL_CHECK_PEER_NAME=true
REM Override Key/Trust Store types if necessary
REM set SOLR_SSL_KEY_STORE_TYPE=PKCS12
REM set SOLR_SSL_TRUST_STORE_TYPE=PKCS12
REM Uncomment if you want to override previously defined SSL values for HTTP client
REM otherwise keep them commented and the above values will automatically be set for HTTP clients
REM set SOLR_SSL_CLIENT_KEY_STORE=
REM set SOLR_SSL_CLIENT_KEY_STORE_PASSWORD=
REM set SOLR_SSL_CLIENT_TRUST_STORE=
REM set SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=
REM set SOLR_SSL_CLIENT_KEY_STORE_TYPE=
REM set SOLR_SSL_CLIENT_TRUST_STORE_TYPE=
REM Sets path of Hadoop credential provider (hadoop.security.credential.provider.path property) and
REM enables usage of credential store.
REM Credential provider should store the following keys:
REM * solr.jetty.keystore.password
REM * solr.jetty.truststore.password
REM Set the two below if you want to set specific store passwords for HTTP client
REM * javax.net.ssl.keyStorePassword
REM * javax.net.ssl.trustStorePassword
REM More info: https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html
REM set SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH=localjceks://file/home/solr/hadoop-credential-provider.jceks
REM set SOLR_OPTS=" -Dsolr.ssl.credential.provider.chain=hadoop"
REM Settings for authentication
REM Please configure only one of SOLR_AUTHENTICATION_CLIENT_BUILDER or SOLR_AUTH_TYPE parameters
REM set SOLR_AUTHENTICATION_CLIENT_BUILDER=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory
REM set SOLR_AUTH_TYPE=basic
REM set SOLR_AUTHENTICATION_OPTS="-Dbasicauth=solr:SolrRocks"
REM Settings for ZK ACL
REM set SOLR_ZK_CREDS_AND_ACLS=-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider ^
REM -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider ^
REM -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD ^
REM -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD
REM set SOLR_OPTS=%SOLR_OPTS% %SOLR_ZK_CREDS_AND_ACLS%
REM When running Solr in non-cloud mode and if planning to do distributed search (using the "shards" parameter), the
REM list of hosts needs to be whitelisted or Solr will forbid the request. The whitelist can be configured in solr.xml,
REM or if you are using the OOTB solr.xml, can be specified using the system property "solr.shardsWhitelist". Alternatively
REM host checking can be disabled by using the system property "solr.disable.shardsWhitelist"
REM set SOLR_OPTS="%SOLR_OPTS% -Dsolr.shardsWhitelist=http://localhost:8983,http://localhost:8984"
REM For a visual indication in the Admin UI of what type of environment this cluster is, configure
REM a -Dsolr.environment property below. Valid values are prod, stage, test, dev, with an optional
REM label or color, e.g. -Dsolr.environment=test,label=Functional+test,color=brown
REM SOLR_OPTS="$SOLR_OPTS -Dsolr.environment=prod"
REM Runs solr in a java security manager sandbox. This can protect against some attacks.
REM Runtime properties are passed to the security policy file (server\etc\security.policy)
REM You can also tweak via standard JDK files such as ~\.java.policy, see https://s.apache.org/java8policy
REM This is experimental! It may not work at all with Hadoop/HDFS features.
REM set SOLR_SECURITY_MANAGER_ENABLED=false

172
solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java
View File

@ -1,86 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.solr.client.solrj.cloud.ShardTerms;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrCore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Start recovery of a core if its term is less than leader's term
*/
public class RecoveringCoreTermWatcher implements ZkShardTerms.CoreTermWatcher {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final CoreDescriptor coreDescriptor;
private final CoreContainer coreContainer;
// used to prevent the case when term of other replicas get changed, we redo recovery
// the idea here is with a specific term of a replica, we only do recovery one
private final AtomicLong lastTermDoRecovery;
RecoveringCoreTermWatcher(CoreDescriptor coreDescriptor, CoreContainer coreContainer) {
this.coreDescriptor = coreDescriptor;
this.coreContainer = coreContainer;
this.lastTermDoRecovery = new AtomicLong(-1);
}
@Override
public boolean onTermChanged(ShardTerms terms) {
if (coreContainer.isShutDown()) return false;
try (SolrCore solrCore = coreContainer.getCore(coreDescriptor.getName())) {
if (solrCore == null || solrCore.isClosed()) {
return false;
}
if (solrCore.getCoreDescriptor() == null || solrCore.getCoreDescriptor().getCloudDescriptor() == null) return true;
String coreNodeName = solrCore.getCoreDescriptor().getCloudDescriptor().getCoreNodeName();
if (terms.haveHighestTermValue(coreNodeName)) return true;
if (lastTermDoRecovery.get() < terms.getTerm(coreNodeName)) {
log.info("Start recovery on {} because core's term is less than leader's term", coreNodeName);
lastTermDoRecovery.set(terms.getTerm(coreNodeName));
solrCore.getUpdateHandler().getSolrCoreState().doRecovery(solrCore.getCoreContainer(), solrCore.getCoreDescriptor());
}
} catch (Exception e) {
log.info("Failed to watch term of core {}", coreDescriptor.getName(), e);
return false;
}
return true;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
RecoveringCoreTermWatcher that = (RecoveringCoreTermWatcher) o;
return coreDescriptor.getName().equals(that.coreDescriptor.getName());
}
@Override
public int hashCode() {
return coreDescriptor.getName().hashCode();
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.solr.client.solrj.cloud.ShardTerms;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrCore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Start recovery of a core if its term is less than leader's term
*/
public class RecoveringCoreTermWatcher implements ZkShardTerms.CoreTermWatcher {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final CoreDescriptor coreDescriptor;
private final CoreContainer coreContainer;
// used to prevent the case when term of other replicas get changed, we redo recovery
// the idea here is with a specific term of a replica, we only do recovery one
private final AtomicLong lastTermDoRecovery;
RecoveringCoreTermWatcher(CoreDescriptor coreDescriptor, CoreContainer coreContainer) {
this.coreDescriptor = coreDescriptor;
this.coreContainer = coreContainer;
this.lastTermDoRecovery = new AtomicLong(-1);
}
@Override
public boolean onTermChanged(ShardTerms terms) {
if (coreContainer.isShutDown()) return false;
try (SolrCore solrCore = coreContainer.getCore(coreDescriptor.getName())) {
if (solrCore == null || solrCore.isClosed()) {
return false;
}
if (solrCore.getCoreDescriptor() == null || solrCore.getCoreDescriptor().getCloudDescriptor() == null) return true;
String coreNodeName = solrCore.getCoreDescriptor().getCloudDescriptor().getCoreNodeName();
if (terms.haveHighestTermValue(coreNodeName)) return true;
if (lastTermDoRecovery.get() < terms.getTerm(coreNodeName)) {
log.info("Start recovery on {} because core's term is less than leader's term", coreNodeName);
lastTermDoRecovery.set(terms.getTerm(coreNodeName));
solrCore.getUpdateHandler().getSolrCoreState().doRecovery(solrCore.getCoreContainer(), solrCore.getCoreDescriptor());
}
} catch (Exception e) {
log.info("Failed to watch term of core {}", coreDescriptor.getName(), e);
return false;
}
return true;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
RecoveringCoreTermWatcher that = (RecoveringCoreTermWatcher) o;
return coreDescriptor.getName().equals(that.coreDescriptor.getName());
}
@Override
public int hashCode() {
return coreDescriptor.getName().hashCode();
}
}

130
solr/core/src/java/org/apache/solr/cloud/ZkCollectionTerms.java
View File

@ -1,65 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import java.util.HashMap;
import java.util.Map;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.core.CoreDescriptor;
/**
* Used to manage all ZkShardTerms of a collection
*/
class ZkCollectionTerms implements AutoCloseable {
private final String collection;
private final Map<String, ZkShardTerms> terms;
private final SolrZkClient zkClient;
ZkCollectionTerms(String collection, SolrZkClient client) {
this.collection = collection;
this.terms = new HashMap<>();
this.zkClient = client;
ObjectReleaseTracker.track(this);
}
public ZkShardTerms getShard(String shardId) {
synchronized (terms) {
if (!terms.containsKey(shardId)) terms.put(shardId, new ZkShardTerms(collection, shardId, zkClient));
return terms.get(shardId);
}
}
public void remove(String shardId, CoreDescriptor coreDescriptor) {
synchronized (terms) {
if (getShard(shardId).removeTerm(coreDescriptor)) {
terms.remove(shardId).close();
}
}
}
public void close() {
synchronized (terms) {
terms.values().forEach(ZkShardTerms::close);
}
ObjectReleaseTracker.release(this);
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import java.util.HashMap;
import java.util.Map;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.core.CoreDescriptor;
/**
* Used to manage all ZkShardTerms of a collection
*/
class ZkCollectionTerms implements AutoCloseable {
private final String collection;
private final Map<String, ZkShardTerms> terms;
private final SolrZkClient zkClient;
ZkCollectionTerms(String collection, SolrZkClient client) {
this.collection = collection;
this.terms = new HashMap<>();
this.zkClient = client;
ObjectReleaseTracker.track(this);
}
public ZkShardTerms getShard(String shardId) {
synchronized (terms) {
if (!terms.containsKey(shardId)) terms.put(shardId, new ZkShardTerms(collection, shardId, zkClient));
return terms.get(shardId);
}
}
public void remove(String shardId, CoreDescriptor coreDescriptor) {
synchronized (terms) {
if (getShard(shardId).removeTerm(coreDescriptor)) {
terms.remove(shardId).close();
}
}
}
public void close() {
synchronized (terms) {
terms.values().forEach(ZkShardTerms::close);
}
ObjectReleaseTracker.release(this);
}
}

850
solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java
View File

@ -1,425 +1,425 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.solr.client.solrj.cloud.ShardTerms;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreDescriptor;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Class used for interact with a ZK term node.
* Each ZK term node relates to a shard of a collection and have this format (in json)
* <p>
* <code>
* {
* "replicaNodeName1" : 1,
* "replicaNodeName2" : 2,
* ..
* }
* </code>
* <p>
* The values correspond to replicas are called terms.
* Only replicas with highest term value are considered up to date and be able to become leader and serve queries.
* <p>
* Terms can only updated in two strict ways:
* <ul>
* <li>A replica sets its term equals to leader's term
* <li>The leader increase its term and some other replicas by 1
* </ul>
* This class should not be reused after {@link org.apache.zookeeper.Watcher.Event.KeeperState#Expired} event
*/
public class ZkShardTerms implements AutoCloseable{
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final Object writingLock = new Object();
private final String collection;
private final String shard;
private final String znodePath;
private final SolrZkClient zkClient;
private final Set<CoreTermWatcher> listeners = new HashSet<>();
private final AtomicBoolean isClosed = new AtomicBoolean(false);
private ShardTerms terms;
// Listener of a core for shard's term change events
interface CoreTermWatcher {
// return true if the listener wanna to be triggered in the next time
boolean onTermChanged(ShardTerms terms);
}
public ZkShardTerms(String collection, String shard, SolrZkClient zkClient) {
this.znodePath = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/terms/" + shard;
this.collection = collection;
this.shard = shard;
this.zkClient = zkClient;
ensureTermNodeExist();
refreshTerms();
retryRegisterWatcher();
ObjectReleaseTracker.track(this);
}
/**
* Ensure that leader's term is higher than some replica's terms
* @param leader coreNodeName of leader
* @param replicasNeedingRecovery set of replicas in which their terms should be lower than leader's term
*/
public void ensureTermsIsHigher(String leader, Set<String> replicasNeedingRecovery) {
if (replicasNeedingRecovery.isEmpty()) return;
ShardTerms newTerms;
while( (newTerms = terms.increaseTerms(leader, replicasNeedingRecovery)) != null) {
if (forceSaveTerms(newTerms)) return;
}
}
public ShardTerms getShardTerms() {
return terms;
}
/**
* Can this replica become leader?
* @param coreNodeName of the replica
* @return true if this replica can become leader, false if otherwise
*/
public boolean canBecomeLeader(String coreNodeName) {
return terms.canBecomeLeader(coreNodeName);
}
/**
* Should leader skip sending updates to this replica?
* @param coreNodeName of the replica
* @return true if this replica has term equals to leader's term, false if otherwise
*/
public boolean skipSendingUpdatesTo(String coreNodeName) {
return !terms.haveHighestTermValue(coreNodeName);
}
/**
* Did this replica registered its term? This is a sign to check f
* @param coreNodeName of the replica
* @return true if this replica registered its term, false if otherwise
*/
public boolean registered(String coreNodeName) {
return terms.getTerm(coreNodeName) != null;
}
public void close() {
// no watcher will be registered
isClosed.set(true);
synchronized (listeners) {
listeners.clear();
}
ObjectReleaseTracker.release(this);
}
// package private for testing, only used by tests
Map<String, Long> getTerms() {
synchronized (writingLock) {
return terms.getTerms();
}
}
/**
* Add a listener so the next time the shard's term get updated, listeners will be called
*/
void addListener(CoreTermWatcher listener) {
synchronized (listeners) {
listeners.add(listener);
}
}
/**
* Remove the coreNodeName from terms map and also remove any expired listeners
* @return Return true if this object should not be reused
*/
boolean removeTerm(CoreDescriptor cd) {
int numListeners;
synchronized (listeners) {
// solrcore already closed
listeners.removeIf(coreTermWatcher -> !coreTermWatcher.onTermChanged(terms));
numListeners = listeners.size();
}
return removeTerm(cd.getCloudDescriptor().getCoreNodeName()) || numListeners == 0;
}
// package private for testing, only used by tests
// return true if this object should not be reused
boolean removeTerm(String coreNodeName) {
ShardTerms newTerms;
while ( (newTerms = terms.removeTerm(coreNodeName)) != null) {
try {
if (saveTerms(newTerms)) return false;
} catch (KeeperException.NoNodeException e) {
return true;
}
}
return true;
}
/**
* Register a replica's term (term value will be 0).
* If a term is already associate with this replica do nothing
* @param coreNodeName of the replica
*/
void registerTerm(String coreNodeName) {
ShardTerms newTerms;
while ( (newTerms = terms.registerTerm(coreNodeName)) != null) {
if (forceSaveTerms(newTerms)) break;
}
}
/**
* Set a replica's term equals to leader's term, and remove recovering flag of a replica.
* This call should only be used by {@link org.apache.solr.common.params.CollectionParams.CollectionAction#FORCELEADER}
* @param coreNodeName of the replica
*/
public void setTermEqualsToLeader(String coreNodeName) {
ShardTerms newTerms;
while ( (newTerms = terms.setTermEqualsToLeader(coreNodeName)) != null) {
if (forceSaveTerms(newTerms)) break;
}
}
public void setTermToZero(String coreNodeName) {
ShardTerms newTerms;
while ( (newTerms = terms.setTermToZero(coreNodeName)) != null) {
if (forceSaveTerms(newTerms)) break;
}
}
/**
* Mark {@code coreNodeName} as recovering
*/
public void startRecovering(String coreNodeName) {
ShardTerms newTerms;
while ( (newTerms = terms.startRecovering(coreNodeName)) != null) {
if (forceSaveTerms(newTerms)) break;
}
}
/**
* Mark {@code coreNodeName} as finished recovering
*/
public void doneRecovering(String coreNodeName) {
ShardTerms newTerms;
while ( (newTerms = terms.doneRecovering(coreNodeName)) != null) {
if (forceSaveTerms(newTerms)) break;
}
}
public boolean isRecovering(String name) {
return terms.isRecovering(name);
}
/**
* When first updates come in, all replicas have some data now,
* so we must switch from term 0 (registered) to 1 (have some data)
*/
public void ensureHighestTermsAreNotZero() {
ShardTerms newTerms;
while ( (newTerms = terms.ensureHighestTermsAreNotZero()) != null) {
if (forceSaveTerms(newTerms)) break;
}
}
public long getHighestTerm() {
return terms.getMaxTerm();
}
public long getTerm(String coreNodeName) {
Long term = terms.getTerm(coreNodeName);
return term == null? -1 : term;
}
// package private for testing, only used by tests
int getNumListeners() {
synchronized (listeners) {
return listeners.size();
}
}
/**
* Set new terms to ZK.
* In case of correspond ZK term node is not created, create it
* @param newTerms to be set
* @return true if terms is saved successfully to ZK, false if otherwise
*/
private boolean forceSaveTerms(ShardTerms newTerms) {
try {
return saveTerms(newTerms);
} catch (KeeperException.NoNodeException e) {
ensureTermNodeExist();
return false;
}
}
/**
* Set new terms to ZK, the version of new terms must match the current ZK term node
* @param newTerms to be set
* @return true if terms is saved successfully to ZK, false if otherwise
* @throws KeeperException.NoNodeException correspond ZK term node is not created
*/
private boolean saveTerms(ShardTerms newTerms) throws KeeperException.NoNodeException {
byte[] znodeData = Utils.toJSON(newTerms);
try {
Stat stat = zkClient.setData(znodePath, znodeData, newTerms.getVersion(), true);
setNewTerms(new ShardTerms(newTerms, stat.getVersion()));
log.info("Successful update of terms at {} to {}", znodePath, newTerms);
return true;
} catch (KeeperException.BadVersionException e) {
log.info("Failed to save terms, version is not a match, retrying");
refreshTerms();
} catch (KeeperException.NoNodeException e) {
throw e;
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error while saving shard term for collection: " + collection, e);
}
return false;
}
/**
* Create correspond ZK term node
*/
private void ensureTermNodeExist() {
String path = "/collections/" + collection + "/terms";
try {
path += "/" + shard;
try {
Map<String,Long> initialTerms = new HashMap<>();
zkClient.makePath(path, Utils.toJSON(initialTerms), CreateMode.PERSISTENT, true);
} catch (KeeperException.NodeExistsException e) {
// it's okay if another beats us creating the node
}
} catch (InterruptedException e) {
Thread.interrupted();
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error creating shard term node in Zookeeper for collection: " + collection, e);
} catch (KeeperException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error creating shard term node in Zookeeper for collection: " + collection, e);
}
}
/**
* Fetch latest terms from ZK
*/
public void refreshTerms() {
ShardTerms newTerms;
try {
Stat stat = new Stat();
byte[] data = zkClient.getData(znodePath, null, stat, true);
newTerms = new ShardTerms((Map<String, Long>) Utils.fromJSON(data), stat.getVersion());
} catch (KeeperException e) {
Thread.interrupted();
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error updating shard term for collection: " + collection, e);
} catch (InterruptedException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error updating shard term for collection: " + collection, e);
}
setNewTerms(newTerms);
}
/**
* Retry register a watcher to the correspond ZK term node
*/
private void retryRegisterWatcher() {
while (!isClosed.get()) {
try {
registerWatcher();
return;
} catch (KeeperException.SessionExpiredException | KeeperException.AuthFailedException e) {
isClosed.set(true);
log.error("Failed watching shard term for collection: {} due to unrecoverable exception", collection, e);
return;
} catch (KeeperException e) {
log.warn("Failed watching shard term for collection: {}, retrying!", collection, e);
try {
zkClient.getConnectionManager().waitForConnected(zkClient.getZkClientTimeout());
} catch (TimeoutException te) {
if (Thread.interrupted()) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error watching shard term for collection: " + collection, te);
}
}
}
}
}
/**
* Register a watcher to the correspond ZK term node
*/
private void registerWatcher() throws KeeperException {
Watcher watcher = event -> {
// session events are not change events, and do not remove the watcher
if (Watcher.Event.EventType.None == event.getType()) {
return;
}
retryRegisterWatcher();
// Some events may be missed during register a watcher, so it is safer to refresh terms after registering watcher
refreshTerms();
};
try {
// exists operation is faster than getData operation
zkClient.exists(znodePath, watcher, true);
} catch (InterruptedException e) {
Thread.interrupted();
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error watching shard term for collection: " + collection, e);
}
}
/**
* Atomically update {@link ZkShardTerms#terms} and call listeners
* @param newTerms to be set
*/
private void setNewTerms(ShardTerms newTerms) {
boolean isChanged = false;
synchronized (writingLock) {
if (terms == null || newTerms.getVersion() > terms.getVersion()) {
terms = newTerms;
isChanged = true;
}
}
if (isChanged) onTermUpdates(newTerms);
}
private void onTermUpdates(ShardTerms newTerms) {
synchronized (listeners) {
listeners.removeIf(coreTermWatcher -> !coreTermWatcher.onTermChanged(newTerms));
}
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.solr.client.solrj.cloud.ShardTerms;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreDescriptor;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Class used for interact with a ZK term node.
* Each ZK term node relates to a shard of a collection and have this format (in json)
* <p>
* <code>
* {
* "replicaNodeName1" : 1,
* "replicaNodeName2" : 2,
* ..
* }
* </code>
* <p>
* The values correspond to replicas are called terms.
* Only replicas with highest term value are considered up to date and be able to become leader and serve queries.
* <p>
* Terms can only updated in two strict ways:
* <ul>
* <li>A replica sets its term equals to leader's term
* <li>The leader increase its term and some other replicas by 1
* </ul>
* This class should not be reused after {@link org.apache.zookeeper.Watcher.Event.KeeperState#Expired} event
*/
public class ZkShardTerms implements AutoCloseable{
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final Object writingLock = new Object();
private final String collection;
private final String shard;
private final String znodePath;
private final SolrZkClient zkClient;
private final Set<CoreTermWatcher> listeners = new HashSet<>();
private final AtomicBoolean isClosed = new AtomicBoolean(false);
private ShardTerms terms;
// Listener of a core for shard's term change events
interface CoreTermWatcher {
// return true if the listener wanna to be triggered in the next time
boolean onTermChanged(ShardTerms terms);
}
public ZkShardTerms(String collection, String shard, SolrZkClient zkClient) {
this.znodePath = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/terms/" + shard;
this.collection = collection;
this.shard = shard;
this.zkClient = zkClient;
ensureTermNodeExist();
refreshTerms();
retryRegisterWatcher();
ObjectReleaseTracker.track(this);
}
/**
* Ensure that leader's term is higher than some replica's terms
* @param leader coreNodeName of leader
* @param replicasNeedingRecovery set of replicas in which their terms should be lower than leader's term
*/
public void ensureTermsIsHigher(String leader, Set<String> replicasNeedingRecovery) {
if (replicasNeedingRecovery.isEmpty()) return;
ShardTerms newTerms;
while( (newTerms = terms.increaseTerms(leader, replicasNeedingRecovery)) != null) {
if (forceSaveTerms(newTerms)) return;
}
}
public ShardTerms getShardTerms() {
return terms;
}
/**
* Can this replica become leader?
* @param coreNodeName of the replica
* @return true if this replica can become leader, false if otherwise
*/
public boolean canBecomeLeader(String coreNodeName) {
return terms.canBecomeLeader(coreNodeName);
}
/**
* Should leader skip sending updates to this replica?
* @param coreNodeName of the replica
* @return true if this replica has term equals to leader's term, false if otherwise
*/
public boolean skipSendingUpdatesTo(String coreNodeName) {
return !terms.haveHighestTermValue(coreNodeName);
}
/**
* Did this replica registered its term? This is a sign to check f
* @param coreNodeName of the replica
* @return true if this replica registered its term, false if otherwise
*/
public boolean registered(String coreNodeName) {
return terms.getTerm(coreNodeName) != null;
}
public void close() {
// no watcher will be registered
isClosed.set(true);
synchronized (listeners) {
listeners.clear();
}
ObjectReleaseTracker.release(this);
}
// package private for testing, only used by tests
Map<String, Long> getTerms() {
synchronized (writingLock) {
return terms.getTerms();
}
}
/**
* Add a listener so the next time the shard's term get updated, listeners will be called
*/
void addListener(CoreTermWatcher listener) {
synchronized (listeners) {
listeners.add(listener);
}
}
/**
* Remove the coreNodeName from terms map and also remove any expired listeners
* @return Return true if this object should not be reused
*/
boolean removeTerm(CoreDescriptor cd) {
int numListeners;
synchronized (listeners) {
// solrcore already closed
listeners.removeIf(coreTermWatcher -> !coreTermWatcher.onTermChanged(terms));
numListeners = listeners.size();
}
return removeTerm(cd.getCloudDescriptor().getCoreNodeName()) || numListeners == 0;
}
// package private for testing, only used by tests
// return true if this object should not be reused
boolean removeTerm(String coreNodeName) {
ShardTerms newTerms;
while ( (newTerms = terms.removeTerm(coreNodeName)) != null) {
try {
if (saveTerms(newTerms)) return false;
} catch (KeeperException.NoNodeException e) {
return true;
}
}
return true;
}
/**
* Register a replica's term (term value will be 0).
* If a term is already associate with this replica do nothing
* @param coreNodeName of the replica
*/
void registerTerm(String coreNodeName) {
ShardTerms newTerms;
while ( (newTerms = terms.registerTerm(coreNodeName)) != null) {
if (forceSaveTerms(newTerms)) break;
}
}
/**
* Set a replica's term equals to leader's term, and remove recovering flag of a replica.
* This call should only be used by {@link org.apache.solr.common.params.CollectionParams.CollectionAction#FORCELEADER}
* @param coreNodeName of the replica
*/
public void setTermEqualsToLeader(String coreNodeName) {
ShardTerms newTerms;
while ( (newTerms = terms.setTermEqualsToLeader(coreNodeName)) != null) {
if (forceSaveTerms(newTerms)) break;
}
}
public void setTermToZero(String coreNodeName) {
ShardTerms newTerms;
while ( (newTerms = terms.setTermToZero(coreNodeName)) != null) {
if (forceSaveTerms(newTerms)) break;
}
}
/**
* Mark {@code coreNodeName} as recovering
*/
public void startRecovering(String coreNodeName) {
ShardTerms newTerms;
while ( (newTerms = terms.startRecovering(coreNodeName)) != null) {
if (forceSaveTerms(newTerms)) break;
}
}
/**
* Mark {@code coreNodeName} as finished recovering
*/
public void doneRecovering(String coreNodeName) {
ShardTerms newTerms;
while ( (newTerms = terms.doneRecovering(coreNodeName)) != null) {
if (forceSaveTerms(newTerms)) break;
}
}
public boolean isRecovering(String name) {
return terms.isRecovering(name);
}
/**
* When first updates come in, all replicas have some data now,
* so we must switch from term 0 (registered) to 1 (have some data)
*/
public void ensureHighestTermsAreNotZero() {
ShardTerms newTerms;
while ( (newTerms = terms.ensureHighestTermsAreNotZero()) != null) {
if (forceSaveTerms(newTerms)) break;
}
}
public long getHighestTerm() {
return terms.getMaxTerm();
}
public long getTerm(String coreNodeName) {
Long term = terms.getTerm(coreNodeName);
return term == null? -1 : term;
}
// package private for testing, only used by tests
int getNumListeners() {
synchronized (listeners) {
return listeners.size();
}
}
/**
* Set new terms to ZK.
* In case of correspond ZK term node is not created, create it
* @param newTerms to be set
* @return true if terms is saved successfully to ZK, false if otherwise
*/
private boolean forceSaveTerms(ShardTerms newTerms) {
try {
return saveTerms(newTerms);
} catch (KeeperException.NoNodeException e) {
ensureTermNodeExist();
return false;
}
}
/**
* Set new terms to ZK, the version of new terms must match the current ZK term node
* @param newTerms to be set
* @return true if terms is saved successfully to ZK, false if otherwise
* @throws KeeperException.NoNodeException correspond ZK term node is not created
*/
private boolean saveTerms(ShardTerms newTerms) throws KeeperException.NoNodeException {
byte[] znodeData = Utils.toJSON(newTerms);
try {
Stat stat = zkClient.setData(znodePath, znodeData, newTerms.getVersion(), true);
setNewTerms(new ShardTerms(newTerms, stat.getVersion()));
log.info("Successful update of terms at {} to {}", znodePath, newTerms);
return true;
} catch (KeeperException.BadVersionException e) {
log.info("Failed to save terms, version is not a match, retrying");
refreshTerms();
} catch (KeeperException.NoNodeException e) {
throw e;
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error while saving shard term for collection: " + collection, e);
}
return false;
}
/**
* Create correspond ZK term node
*/
private void ensureTermNodeExist() {
String path = "/collections/" + collection + "/terms";
try {
path += "/" + shard;
try {
Map<String,Long> initialTerms = new HashMap<>();
zkClient.makePath(path, Utils.toJSON(initialTerms), CreateMode.PERSISTENT, true);
} catch (KeeperException.NodeExistsException e) {
// it's okay if another beats us creating the node
}
} catch (InterruptedException e) {
Thread.interrupted();
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error creating shard term node in Zookeeper for collection: " + collection, e);
} catch (KeeperException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Error creating shard term node in Zookeeper for collection: " + collection, e);
}
}
/**
* Fetch latest terms from ZK
*/
public void refreshTerms() {
ShardTerms newTerms;
try {
Stat stat = new Stat();
byte[] data = zkClient.getData(znodePath, null, stat, true);
newTerms = new ShardTerms((Map<String, Long>) Utils.fromJSON(data), stat.getVersion());
} catch (KeeperException e) {
Thread.interrupted();
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error updating shard term for collection: " + collection, e);
} catch (InterruptedException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error updating shard term for collection: " + collection, e);
}
setNewTerms(newTerms);
}
/**
* Retry register a watcher to the correspond ZK term node
*/
private void retryRegisterWatcher() {
while (!isClosed.get()) {
try {
registerWatcher();
return;
} catch (KeeperException.SessionExpiredException | KeeperException.AuthFailedException e) {
isClosed.set(true);
log.error("Failed watching shard term for collection: {} due to unrecoverable exception", collection, e);
return;
} catch (KeeperException e) {
log.warn("Failed watching shard term for collection: {}, retrying!", collection, e);
try {
zkClient.getConnectionManager().waitForConnected(zkClient.getZkClientTimeout());
} catch (TimeoutException te) {
if (Thread.interrupted()) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error watching shard term for collection: " + collection, te);
}
}
}
}
}
/**
* Register a watcher to the correspond ZK term node
*/
private void registerWatcher() throws KeeperException {
Watcher watcher = event -> {
// session events are not change events, and do not remove the watcher
if (Watcher.Event.EventType.None == event.getType()) {
return;
}
retryRegisterWatcher();
// Some events may be missed during register a watcher, so it is safer to refresh terms after registering watcher
refreshTerms();
};
try {
// exists operation is faster than getData operation
zkClient.exists(znodePath, watcher, true);
} catch (InterruptedException e) {
Thread.interrupted();
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error watching shard term for collection: " + collection, e);
}
}
/**
* Atomically update {@link ZkShardTerms#terms} and call listeners
* @param newTerms to be set
*/
private void setNewTerms(ShardTerms newTerms) {
boolean isChanged = false;
synchronized (writingLock) {
if (terms == null || newTerms.getVersion() > terms.getVersion()) {
terms = newTerms;
isChanged = true;
}
}
if (isChanged) onTermUpdates(newTerms);
}
private void onTermUpdates(ShardTerms newTerms) {
synchronized (listeners) {
listeners.removeIf(coreTermWatcher -> !coreTermWatcher.onTermChanged(newTerms));
}
}
}

432
solr/core/src/java/org/apache/solr/highlight/LuceneRegexFragmenter.java
View File

@ -1,217 +1,217 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.highlight;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.search.highlight.Fragmenter;
/**
* Fragmenter that tries to produce snippets that "look" like a regular
* expression.
*
* NOTE: the default for <code>maxAnalyzedChars</code> is much lower for this
* fragmenter. After this limit is exhausted, fragments are produced in the
* same way as <code>GapFragmenter</code>
*/
class LuceneRegexFragmenter implements Fragmenter
{
// ** defaults
public static final int DEFAULT_FRAGMENT_SIZE = 70;
public static final int DEFAULT_INCREMENT_GAP = 50;
public static final float DEFAULT_SLOP = 0.6f;
public static final int DEFAULT_MAX_ANALYZED_CHARS = 10000;
// ** settings
// desired length of fragments, in characters
protected int targetFragChars;
// increment gap which indicates a new fragment should occur
// (often due to multi-valued fields)
protected int incrementGapThreshold;
// factor by which we are allowed to bend the frag size (larger or smaller)
protected float slop;
// analysis limit (ensures we don't waste too much time on long fields)
protected int maxAnalyzedChars;
// default desirable pattern for text fragments.
protected Pattern textRE;
// ** state
protected int currentNumFrags;
protected int currentOffset;
protected int targetOffset;
protected int[] hotspots;
private PositionIncrementAttribute posIncAtt;
private OffsetAttribute offsetAtt;
// ** other
// note: could dynamically change size of sentences extracted to match
// target frag size
public static final String
DEFAULT_PATTERN_RAW = "[-\\w ,\\n\"']{20,200}";
public static final Pattern
DEFAULT_PATTERN = Pattern.compile(DEFAULT_PATTERN_RAW);
public LuceneRegexFragmenter() {
this(DEFAULT_FRAGMENT_SIZE,
DEFAULT_INCREMENT_GAP,
DEFAULT_SLOP,
DEFAULT_MAX_ANALYZED_CHARS);
}
public LuceneRegexFragmenter(int targetFragChars) {
this(targetFragChars,
DEFAULT_INCREMENT_GAP,
DEFAULT_SLOP,
DEFAULT_MAX_ANALYZED_CHARS);
}
public LuceneRegexFragmenter(int targetFragChars,
int incrementGapThreshold,
float slop,
int maxAnalyzedChars ) {
this(targetFragChars, incrementGapThreshold, slop, maxAnalyzedChars,
DEFAULT_PATTERN);
}
public LuceneRegexFragmenter(int targetFragChars,
int incrementGapThreshold,
float slop,
int maxAnalyzedChars,
Pattern targetPattern) {
this.targetFragChars = targetFragChars;
this.incrementGapThreshold = incrementGapThreshold;
this.slop = slop;
this.maxAnalyzedChars = maxAnalyzedChars;
this.textRE = targetPattern;
}
/* (non-Javadoc)
* @see org.apache.lucene.search.highlight.TextFragmenter#start(java.lang.String)
*/
@Override
public void start(String originalText, TokenStream tokenStream) {
currentNumFrags = 1;
currentOffset = 0;
addHotSpots(originalText);
posIncAtt = tokenStream.getAttribute(PositionIncrementAttribute.class);
offsetAtt = tokenStream.getAttribute(OffsetAttribute.class);
}
////////////////////////////////////
// pre-analysis
////////////////////////////////////
protected void addHotSpots(String text) {
//System.out.println("hot spotting");
ArrayList<Integer> temphs = new ArrayList<>(
text.length() / targetFragChars);
Matcher match = textRE.matcher(text);
int cur = 0;
while(match.find() && cur < maxAnalyzedChars) {
int start=match.start(), end=match.end();
temphs.add(start);
temphs.add(end);
cur = end;
//System.out.println("Matched " + match.group());
}
hotspots = new int[temphs.size()];
for(int i = 0; i < temphs.size(); i++) {
hotspots[i] = temphs.get(i);
}
// perhaps not necessary--I don't know if re matches are non-overlapping
Arrays.sort(hotspots);
}
////////////////////////////////////
// fragmenting
////////////////////////////////////
/* (non-Javadoc)
* @see org.apache.lucene.search.highlight.TextFragmenter#isNewFragment(org.apache.lucene.analysis.Token)
*/
@Override
public boolean isNewFragment()
{
boolean isNewFrag = false;
int minFragLen = (int)((1.0f - slop)*targetFragChars);
int endOffset = offsetAtt.endOffset();
// ** determin isNewFrag
if(posIncAtt.getPositionIncrement() > incrementGapThreshold) {
// large position gaps always imply new fragments
isNewFrag = true;
} else if(endOffset - currentOffset < minFragLen) {
// we're not in our range of flexibility
isNewFrag = false;
} else if(targetOffset > 0) {
// we've already decided on a target
isNewFrag = endOffset > targetOffset;
} else {
// we might be able to do something
int minOffset = currentOffset + minFragLen;
int maxOffset = (int)(currentOffset + (1.0f + slop)*targetFragChars);
int hotIndex;
// look for a close hotspot
hotIndex = Arrays.binarySearch(hotspots, endOffset);
if(hotIndex < 0) hotIndex = -hotIndex;
if(hotIndex >= hotspots.length) {
// no more hotspots in this input stream
targetOffset = currentOffset + targetFragChars;
} else if(hotspots[hotIndex] > maxOffset) {
// no hotspots within slop
targetOffset = currentOffset + targetFragChars;
} else {
// try to find hotspot in slop
int goal = hotspots[hotIndex];
while(goal < minOffset && hotIndex < hotspots.length) {
hotIndex++;
goal = hotspots[hotIndex];
}
targetOffset = goal <= maxOffset ? goal : currentOffset + targetFragChars;
}
isNewFrag = endOffset > targetOffset;
}
// ** operate on isNewFrag
if(isNewFrag) {
currentNumFrags++;
currentOffset = endOffset;
targetOffset = -1;
}
return isNewFrag;
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.highlight;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.search.highlight.Fragmenter;
/**
* Fragmenter that tries to produce snippets that "look" like a regular
* expression.
*
* NOTE: the default for <code>maxAnalyzedChars</code> is much lower for this
* fragmenter. After this limit is exhausted, fragments are produced in the
* same way as <code>GapFragmenter</code>
*/
class LuceneRegexFragmenter implements Fragmenter
{
// ** defaults
public static final int DEFAULT_FRAGMENT_SIZE = 70;
public static final int DEFAULT_INCREMENT_GAP = 50;
public static final float DEFAULT_SLOP = 0.6f;
public static final int DEFAULT_MAX_ANALYZED_CHARS = 10000;
// ** settings
// desired length of fragments, in characters
protected int targetFragChars;
// increment gap which indicates a new fragment should occur
// (often due to multi-valued fields)
protected int incrementGapThreshold;
// factor by which we are allowed to bend the frag size (larger or smaller)
protected float slop;
// analysis limit (ensures we don't waste too much time on long fields)
protected int maxAnalyzedChars;
// default desirable pattern for text fragments.
protected Pattern textRE;
// ** state
protected int currentNumFrags;
protected int currentOffset;
protected int targetOffset;
protected int[] hotspots;
private PositionIncrementAttribute posIncAtt;
private OffsetAttribute offsetAtt;
// ** other
// note: could dynamically change size of sentences extracted to match
// target frag size
public static final String
DEFAULT_PATTERN_RAW = "[-\\w ,\\n\"']{20,200}";
public static final Pattern
DEFAULT_PATTERN = Pattern.compile(DEFAULT_PATTERN_RAW);
public LuceneRegexFragmenter() {
this(DEFAULT_FRAGMENT_SIZE,
DEFAULT_INCREMENT_GAP,
DEFAULT_SLOP,
DEFAULT_MAX_ANALYZED_CHARS);
}
public LuceneRegexFragmenter(int targetFragChars) {
this(targetFragChars,
DEFAULT_INCREMENT_GAP,
DEFAULT_SLOP,
DEFAULT_MAX_ANALYZED_CHARS);
}
public LuceneRegexFragmenter(int targetFragChars,
int incrementGapThreshold,
float slop,
int maxAnalyzedChars ) {
this(targetFragChars, incrementGapThreshold, slop, maxAnalyzedChars,
DEFAULT_PATTERN);
}
public LuceneRegexFragmenter(int targetFragChars,
int incrementGapThreshold,
float slop,
int maxAnalyzedChars,
Pattern targetPattern) {
this.targetFragChars = targetFragChars;
this.incrementGapThreshold = incrementGapThreshold;
this.slop = slop;
this.maxAnalyzedChars = maxAnalyzedChars;
this.textRE = targetPattern;
}
/* (non-Javadoc)
* @see org.apache.lucene.search.highlight.TextFragmenter#start(java.lang.String)
*/
@Override
public void start(String originalText, TokenStream tokenStream) {
currentNumFrags = 1;
currentOffset = 0;
addHotSpots(originalText);
posIncAtt = tokenStream.getAttribute(PositionIncrementAttribute.class);
offsetAtt = tokenStream.getAttribute(OffsetAttribute.class);
}
////////////////////////////////////
// pre-analysis
////////////////////////////////////
protected void addHotSpots(String text) {
//System.out.println("hot spotting");
ArrayList<Integer> temphs = new ArrayList<>(
text.length() / targetFragChars);
Matcher match = textRE.matcher(text);
int cur = 0;
while(match.find() && cur < maxAnalyzedChars) {
int start=match.start(), end=match.end();
temphs.add(start);
temphs.add(end);
cur = end;
//System.out.println("Matched " + match.group());
}
hotspots = new int[temphs.size()];
for(int i = 0; i < temphs.size(); i++) {
hotspots[i] = temphs.get(i);
}
// perhaps not necessary--I don't know if re matches are non-overlapping
Arrays.sort(hotspots);
}
////////////////////////////////////
// fragmenting
////////////////////////////////////
/* (non-Javadoc)
* @see org.apache.lucene.search.highlight.TextFragmenter#isNewFragment(org.apache.lucene.analysis.Token)
*/
@Override
public boolean isNewFragment()
{
boolean isNewFrag = false;
int minFragLen = (int)((1.0f - slop)*targetFragChars);
int endOffset = offsetAtt.endOffset();
// ** determin isNewFrag
if(posIncAtt.getPositionIncrement() > incrementGapThreshold) {
// large position gaps always imply new fragments
isNewFrag = true;
} else if(endOffset - currentOffset < minFragLen) {
// we're not in our range of flexibility
isNewFrag = false;
} else if(targetOffset > 0) {
// we've already decided on a target
isNewFrag = endOffset > targetOffset;
} else {
// we might be able to do something
int minOffset = currentOffset + minFragLen;
int maxOffset = (int)(currentOffset + (1.0f + slop)*targetFragChars);
int hotIndex;
// look for a close hotspot
hotIndex = Arrays.binarySearch(hotspots, endOffset);
if(hotIndex < 0) hotIndex = -hotIndex;
if(hotIndex >= hotspots.length) {
// no more hotspots in this input stream
targetOffset = currentOffset + targetFragChars;
} else if(hotspots[hotIndex] > maxOffset) {
// no hotspots within slop
targetOffset = currentOffset + targetFragChars;
} else {
// try to find hotspot in slop
int goal = hotspots[hotIndex];
while(goal < minOffset && hotIndex < hotspots.length) {
hotIndex++;
goal = hotspots[hotIndex];
}
targetOffset = goal <= maxOffset ? goal : currentOffset + targetFragChars;
}
isNewFrag = endOffset > targetOffset;
}
// ** operate on isNewFrag
if(isNewFrag) {
currentNumFrags++;
currentOffset = endOffset;
targetOffset = -1;
}
return isNewFrag;
}
}

42
solr/server/scripts/cloud-scripts/zkcli.bat
View File

@ -1,21 +1,21 @@
@echo off
REM You can override pass the following parameters to this script:
REM
set JVM=java
REM Find location of this script
set SDIR=%~dp0
if "%SDIR:~-1%"=="\" set SDIR=%SDIR:~0,-1%
set "LOG4J_CONFIG=file:///%SDIR%\..\..\resources\log4j2-console.xml"
REM Settings for ZK ACL
REM set SOLR_ZK_CREDS_AND_ACLS=-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider ^
REM -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider ^
REM -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD ^
REM -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD
"%JVM%" %SOLR_ZK_CREDS_AND_ACLS% %ZKCLI_JVM_FLAGS% -Dlog4j.configurationFile="%LOG4J_CONFIG%" ^
-classpath "%SDIR%\..\..\solr-webapp\webapp\WEB-INF\lib\*;%SDIR%\..\..\lib\ext\*;%SDIR%\..\..\lib\*" org.apache.solr.cloud.ZkCLI %*
@echo off
REM You can override pass the following parameters to this script:
REM
set JVM=java
REM Find location of this script
set SDIR=%~dp0
if "%SDIR:~-1%"=="\" set SDIR=%SDIR:~0,-1%
set "LOG4J_CONFIG=file:///%SDIR%\..\..\resources\log4j2-console.xml"
REM Settings for ZK ACL
REM set SOLR_ZK_CREDS_AND_ACLS=-DzkACLProvider=org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider ^
REM -DzkCredentialsProvider=org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider ^
REM -DzkDigestUsername=admin-user -DzkDigestPassword=CHANGEME-ADMIN-PASSWORD ^
REM -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEME-READONLY-PASSWORD
"%JVM%" %SOLR_ZK_CREDS_AND_ACLS% %ZKCLI_JVM_FLAGS% -Dlog4j.configurationFile="%LOG4J_CONFIG%" ^
-classpath "%SDIR%\..\..\solr-webapp\webapp\WEB-INF\lib\*;%SDIR%\..\..\lib\ext\*;%SDIR%\..\..\lib\*" org.apache.solr.cloud.ZkCLI %*