Merge remote-tracking branch 'origin/master' into gradle-master

This commit is contained in:
Dawid Weiss 2019-12-20 17:35:40 +01:00
commit 5897b78572
37 changed files with 359 additions and 304 deletions

View File

@ -89,9 +89,7 @@ Optimizations
Bug Fixes Bug Fixes
--------------------- ---------------------
(No changes)
* LUCENE-9055: Fix the detection of lines crossing triangles through edge points.
(Ignacio Vera)
Other Other
--------------------- ---------------------
@ -173,6 +171,11 @@ Bug Fixes
* LUCENE-8996: maxScore was sometimes missing from distributed grouped responses. * LUCENE-8996: maxScore was sometimes missing from distributed grouped responses.
(Julien Massenet, Diego Ceccarelli, Munendra S N, Christine Poerschke) (Julien Massenet, Diego Ceccarelli, Munendra S N, Christine Poerschke)
* LUCENE-9055: Fix the detection of lines crossing triangles through edge points.
(Ignacio Vera)
* LUCENE-9103: Disjunctions can miss some hits in some rare conditions. (Adrien Grand)
Other Other
* LUCENE-8979: Code Cleanup: Use entryset for map iteration wherever possible. - Part 2 (Koen De Groote) * LUCENE-8979: Code Cleanup: Use entryset for map iteration wherever possible. - Part 2 (Koen De Groote)

View File

@ -28,6 +28,7 @@ import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.ByteRunAutomaton; import org.apache.lucene.util.automaton.ByteRunAutomaton;
import org.apache.lucene.util.automaton.LevenshteinAutomata; import org.apache.lucene.util.automaton.LevenshteinAutomata;
import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
/** Implements the fuzzy search query. The similarity measurement /** Implements the fuzzy search query. The similarity measurement
* is based on the Damerau-Levenshtein (optimal string alignment) algorithm, * is based on the Damerau-Levenshtein (optimal string alignment) algorithm,
@ -163,8 +164,12 @@ public class FuzzyQuery extends MultiTermQuery {
visitor.consumeTerms(this, term); visitor.consumeTerms(this, term);
} else { } else {
// Note: we're rebuilding the automaton here, so this can be expensive // Note: we're rebuilding the automaton here, so this can be expensive
visitor.consumeTermsMatching(this, field, try {
new ByteRunAutomaton(toAutomaton(), false, Operations.DEFAULT_MAX_DETERMINIZED_STATES)); visitor.consumeTermsMatching(this, field,
new ByteRunAutomaton(toAutomaton(), false, Operations.DEFAULT_MAX_DETERMINIZED_STATES));
} catch (TooComplexToDeterminizeException e) {
throw new FuzzyTermsEnum.FuzzyTermsException(term.text(), e);
}
} }
} }
} }

View File

@ -37,6 +37,7 @@ import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.LevenshteinAutomata; import org.apache.lucene.util.automaton.LevenshteinAutomata;
import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
/** Subclass of TermsEnum for enumerating all terms that are similar /** Subclass of TermsEnum for enumerating all terms that are similar
* to the specified filter term. * to the specified filter term.
@ -131,7 +132,11 @@ public final class FuzzyTermsEnum extends BaseTermsEnum {
prevAutomata = new CompiledAutomaton[maxEdits+1]; prevAutomata = new CompiledAutomaton[maxEdits+1];
Automaton[] automata = buildAutomata(termText, prefixLength, transpositions, maxEdits); Automaton[] automata = buildAutomata(termText, prefixLength, transpositions, maxEdits);
for (int i = 0; i <= maxEdits; i++) { for (int i = 0; i <= maxEdits; i++) {
prevAutomata[i] = new CompiledAutomaton(automata[i], true, false); try {
prevAutomata[i] = new CompiledAutomaton(automata[i], true, false);
} catch (TooComplexToDeterminizeException e) {
throw new FuzzyTermsException(term.text(), e);
}
} }
// first segment computes the automata, and we share with subsequent segments via this Attribute: // first segment computes the automata, and we share with subsequent segments via this Attribute:
dfaAtt.setAutomata(prevAutomata); dfaAtt.setAutomata(prevAutomata);
@ -407,4 +412,15 @@ public final class FuzzyTermsEnum extends BaseTermsEnum {
reflector.reflect(LevenshteinAutomataAttribute.class, "automata", automata); reflector.reflect(LevenshteinAutomataAttribute.class, "automata", automata);
} }
} }
/**
* Thrown to indicate that there was an issue creating a fuzzy query for a given term.
* Typically occurs with terms longer than 220 UTF-8 characters,
* but also possible with shorter terms consisting of UTF-32 code points.
*/
public static class FuzzyTermsException extends RuntimeException {
FuzzyTermsException(String term, Throwable cause) {
super("Term too complex: " + term, cause);
}
}
} }

View File

@ -185,6 +185,7 @@ final class WANDScorer extends Scorer {
} }
assert minCompetitiveScore == 0 || tailMaxScore < minCompetitiveScore; assert minCompetitiveScore == 0 || tailMaxScore < minCompetitiveScore;
assert doc <= upTo;
return true; return true;
} }
@ -374,17 +375,34 @@ final class WANDScorer extends Scorer {
} }
} }
/**
* Update {@code upTo} and maximum scores of sub scorers so that {@code upTo}
* is greater than or equal to the next candidate after {@code target}, i.e.
* the top of `head`.
*/
private void updateMaxScoresIfNecessary(int target) throws IOException { private void updateMaxScoresIfNecessary(int target) throws IOException {
assert lead == null; assert lead == null;
if (head.size() == 0) { // no matches in the current block while (upTo < DocIdSetIterator.NO_MORE_DOCS) {
if (upTo != DocIdSetIterator.NO_MORE_DOCS) { if (head.size() == 0) {
updateMaxScores(Math.max(target, upTo + 1)); // All clauses could fit in the tail, which means that the sum of the
// maximum scores of sub clauses is less than the minimum competitive score.
// Move to the next block until this condition becomes false.
target = Math.max(target, upTo + 1);
updateMaxScores(target);
} else if (head.top().doc > upTo) {
// We have a next candidate but it's not in the current block. We need to
// move to the next block in order to not miss any potential hits between
// `target` and `head.top().doc`.
assert head.top().doc >= target;
updateMaxScores(target);
break;
} else {
break;
} }
} else if (head.top().doc > upTo) { // the next candidate is in a different block
assert head.top().doc >= target;
updateMaxScores(target);
} }
assert upTo == DocIdSetIterator.NO_MORE_DOCS || (head.size() > 0 && head.top().doc <= upTo);
} }
/** Set 'doc' to the next potential match, and move all disis of 'head' that /** Set 'doc' to the next potential match, and move all disis of 'head' that
@ -394,14 +412,12 @@ final class WANDScorer extends Scorer {
updateMaxScoresIfNecessary(target); updateMaxScoresIfNecessary(target);
assert upTo >= target; assert upTo >= target;
// If the head is empty, it means that the sum of all max scores is not // updateMaxScores tries to move forward until a block with matches is found
// enough to produce a competitive score. So we jump to the next block. // so if the head is empty it means there are no matches at all anymore
while (head.size() == 0) { if (head.size() == 0) {
if (upTo == DocIdSetIterator.NO_MORE_DOCS) { assert upTo == DocIdSetIterator.NO_MORE_DOCS;
doc = DocIdSetIterator.NO_MORE_DOCS; doc = DocIdSetIterator.NO_MORE_DOCS;
return; return;
}
updateMaxScores(upTo + 1);
} }
// The top of `head` defines the next potential match // The top of `head` defines the next potential match

View File

@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
@ -35,6 +36,8 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
@ -43,6 +46,9 @@ import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.automaton.LevenshteinAutomata; import org.apache.lucene.util.automaton.LevenshteinAutomata;
import org.apache.lucene.util.automaton.Operations;
import static org.hamcrest.CoreMatchers.containsString;
/** /**
* Tests {@link FuzzyQuery}. * Tests {@link FuzzyQuery}.
@ -492,7 +498,63 @@ public class TestFuzzyQuery extends LuceneTestCase {
}); });
assertTrue(expected.getMessage().contains("maxExpansions must be positive")); assertTrue(expected.getMessage().contains("maxExpansions must be positive"));
} }
public void testErrorMessage() {
// 45 states per vector from Lev2TParametricDescription
int length = (Operations.DEFAULT_MAX_DETERMINIZED_STATES / 45) + 10;
String value = RandomizedTest.randomRealisticUnicodeOfCodepointLength(length);
FuzzyTermsEnum.FuzzyTermsException expected = expectThrows(FuzzyTermsEnum.FuzzyTermsException.class, () -> {
new FuzzyQuery(new Term("field", value)).getTermsEnum(new Terms() {
@Override
public TermsEnum iterator() {
throw new UnsupportedOperationException();
}
@Override
public long size() {
throw new UnsupportedOperationException();
}
@Override
public long getSumTotalTermFreq() {
throw new UnsupportedOperationException();
}
@Override
public long getSumDocFreq() {
throw new UnsupportedOperationException();
}
@Override
public int getDocCount() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasFreqs() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasOffsets() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasPositions() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasPayloads() {
throw new UnsupportedOperationException();
}
});
});
assertThat(expected.getMessage(), containsString(value));
}
private void addDoc(String text, RandomIndexWriter writer) throws IOException { private void addDoc(String text, RandomIndexWriter writer) throws IOException {
Document doc = new Document(); Document doc = new Document();
doc.add(newTextField("field", text, Field.Store.YES)); doc.add(newTextField("field", text, Field.Store.YES));

View File

@ -61,7 +61,7 @@ public abstract class ReplicatorTestCase extends LuceneTestCase {
// talking to that server, but for the purposes of testing that should // talking to that server, but for the purposes of testing that should
// be good enough // be good enough
final boolean useSsl = Boolean.getBoolean("tests.jettySsl"); final boolean useSsl = Boolean.getBoolean("tests.jettySsl");
final SslContextFactory sslcontext = new SslContextFactory(false); final SslContextFactory.Server sslcontext = new SslContextFactory.Server();
if (useSsl) { if (useSsl) {
if (null != System.getProperty("javax.net.ssl.keyStore")) { if (null != System.getProperty("javax.net.ssl.keyStore")) {

View File

@ -83,7 +83,7 @@ Improvements
* LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via Anshum Gupta) * LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via Anshum Gupta)
* SOLR-13749: New cross collection join filter (XCJF) (Kevin Fox, Kevin Watters, via Gus Heck) * SOLR-13749: New cross collection join filter (XCJF) (Dan Fox, Kevin Watters, via Gus Heck)
Other Changes Other Changes
@ -116,13 +116,16 @@ Upgrade Notes
* SOLR-14092: BlockJoinFacetComponent is marked for deprecation and will be removed in 9.0. * SOLR-14092: BlockJoinFacetComponent is marked for deprecation and will be removed in 9.0.
Users are encouraged to migrate to uniqueBlock() in JSON Facet API. (Mikhail Khludnev) Users are encouraged to migrate to uniqueBlock() in JSON Facet API. (Mikhail Khludnev)
* SOLR-13983: Process execution is removed from SystemInfoHandler. A best-effort attempt to
execute "uname -a" and "uptime" on non-Windows platforms is no longer made. (rmuir)
New Features New Features
--------------------- ---------------------
(No changes) (No changes)
Improvements Improvements
--------------------- ---------------------
(No changes) * SOLR-14042: Fix varargs precommit warnings (Andraas Salamon via Jason Gerlowski)
Optimizations Optimizations
--------------------- ---------------------
@ -133,6 +136,8 @@ Bug Fixes
* SOLR-14099: Fixed @LogLevel annotation in test-framework to correctly 'unset' Loggers after test (hossman) * SOLR-14099: Fixed @LogLevel annotation in test-framework to correctly 'unset' Loggers after test (hossman)
* SOLR-14106: Cleanup Jetty SslContextFactory usage (Ryan Rockenbaugh, Jan Hoydahl, Kevin Risden)
Other Changes Other Changes
--------------------- ---------------------
@ -140,6 +145,7 @@ Other Changes
* SOLR-14054: Upgrade to Tika 1.23 when available (Tim Allison) * SOLR-14054: Upgrade to Tika 1.23 when available (Tim Allison)
* SOLR-14091: Remove deprecated soLingerTime when configuring Jetty connector (Matthias Krueger via Kevin Risden)
================== 8.4.0 ================== ================== 8.4.0 ==================
@ -156,8 +162,8 @@ Jetty 9.4.19.v20190610
Upgrade Notes Upgrade Notes
--------------------- ---------------------
* org.apache.solr.search.grouping.distributed.command.QueryCommand.Builder has new method 'setMainQuery' which is used * SOLR-13823: org.apache.solr.search.grouping.distributed.command.QueryCommand.Builder has new method 'setMainQuery' which is used
to set top-level query. build() would fail if called without setting mainQuery to set top-level query. build() would fail if called without setting mainQuery.
* SOLR-13817: Deprecate legacy SolrCache implementations. Users are encouraged to transition their * SOLR-13817: Deprecate legacy SolrCache implementations. Users are encouraged to transition their
configurations to use org.apache.solr.search.CaffeineCache instead. (ab) configurations to use org.apache.solr.search.CaffeineCache instead. (ab)
@ -267,10 +273,6 @@ Improvements
* SOLR-13970: Fail the request when collapsing or expand is used with Grouping. (Erick Erickson, Joel Bernstein, Munendra S N) * SOLR-13970: Fail the request when collapsing or expand is used with Grouping. (Erick Erickson, Joel Bernstein, Munendra S N)
Optimizations
---------------------
(No changes)
Bug Fixes Bug Fixes
--------------------- ---------------------

View File

@ -152,11 +152,6 @@
</sequential> </sequential>
</macrodef> </macrodef>
<!-- turn on security manager? -->
<condition property="java.security.manager" value="org.apache.solr.util.SolrSecurityManager">
<istrue value="${tests.useSecurityManager}"/>
</condition>
<target name="validate" depends="compile-tools"> <target name="validate" depends="compile-tools">
</target> </target>

View File

@ -278,7 +278,7 @@ public class JettySolrRunner {
// the server as well as any client actions taken by this JVM in // the server as well as any client actions taken by this JVM in
// talking to that server, but for the purposes of testing that should // talking to that server, but for the purposes of testing that should
// be good enough // be good enough
final SslContextFactory sslcontext = SSLConfig.createContextFactory(config.sslConfig); final SslContextFactory.Server sslcontext = SSLConfig.createContextFactory(config.sslConfig);
HttpConfiguration configuration = new HttpConfiguration(); HttpConfiguration configuration = new HttpConfiguration();
ServerConnector connector; ServerConnector connector;
@ -319,7 +319,6 @@ public class JettySolrRunner {
} }
connector.setReuseAddress(true); connector.setReuseAddress(true);
connector.setSoLingerTime(-1);
connector.setPort(port); connector.setPort(port);
connector.setHost("127.0.0.1"); connector.setHost("127.0.0.1");
connector.setIdleTimeout(THREAD_POOL_MAX_IDLE_TIME_MS); connector.setIdleTimeout(THREAD_POOL_MAX_IDLE_TIME_MS);
@ -330,7 +329,6 @@ public class JettySolrRunner {
HttpConfiguration configuration = new HttpConfiguration(); HttpConfiguration configuration = new HttpConfiguration();
ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(configuration)); ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(configuration));
connector.setPort(port); connector.setPort(port);
connector.setSoLingerTime(-1);
connector.setIdleTimeout(THREAD_POOL_MAX_IDLE_TIME_MS); connector.setIdleTimeout(THREAD_POOL_MAX_IDLE_TIME_MS);
server.setConnectors(new Connector[] {connector}); server.setConnectors(new Connector[] {connector});
} }

View File

@ -18,14 +18,11 @@ package org.apache.solr.handler.admin;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles;
import java.lang.management.ManagementFactory; import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean; import java.lang.management.OperatingSystemMXBean;
import java.lang.management.RuntimeMXBean; import java.lang.management.RuntimeMXBean;
import java.net.InetAddress; import java.net.InetAddress;
import java.nio.charset.Charset;
import java.text.DecimalFormat; import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols; import java.text.DecimalFormatSymbols;
import java.util.Date; import java.util.Date;
@ -34,9 +31,7 @@ import java.util.List;
import java.util.Locale; import java.util.Locale;
import com.codahale.metrics.Gauge; import com.codahale.metrics.Gauge;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.LucenePackage; import org.apache.lucene.LucenePackage;
import org.apache.lucene.util.Constants;
import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
@ -227,49 +222,9 @@ public class SystemInfoHandler extends RequestHandlerBase
} }
}); });
// Try some command line things:
try {
if (!Constants.WINDOWS) {
info.add( "uname", execute( "uname -a" ) );
info.add( "uptime", execute( "uptime" ) );
}
} catch( Exception ex ) {
log.warn("Unable to execute command line tools to get operating system properties.", ex);
}
return info; return info;
} }
/**
* Utility function to execute a function
*/
private static String execute( String cmd )
{
InputStream in = null;
Process process = null;
try {
process = Runtime.getRuntime().exec(cmd);
in = process.getInputStream();
// use default charset from locale here, because the command invoked also uses the default locale:
return IOUtils.toString(new InputStreamReader(in, Charset.defaultCharset()));
} catch( Exception ex ) {
// ignore - log.warn("Error executing command", ex);
return "(error executing: " + cmd + ")";
} catch (Error err) {
if (err.getMessage() != null && (err.getMessage().contains("posix_spawn") || err.getMessage().contains("UNIXProcess"))) {
log.warn("Error forking command due to JVM locale bug (see https://issues.apache.org/jira/browse/SOLR-6387): " + err.getMessage());
return "(error executing: " + cmd + ")";
}
throw err;
} finally {
if (process != null) {
IOUtils.closeQuietly( process.getOutputStream() );
IOUtils.closeQuietly( process.getInputStream() );
IOUtils.closeQuietly( process.getErrorStream() );
}
}
}
/** /**
* Get JVM Info - including memory info * Get JVM Info - including memory info
*/ */

View File

@ -37,6 +37,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.FuzzyTermsEnum;
import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query; import org.apache.lucene.search.Query;
@ -1484,7 +1485,11 @@ public class QueryComponent extends SearchComponent
SolrIndexSearcher searcher = req.getSearcher(); SolrIndexSearcher searcher = req.getSearcher();
searcher.search(result, cmd); try {
searcher.search(result, cmd);
} catch (FuzzyTermsEnum.FuzzyTermsException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
rb.setResult(result); rb.setResult(result);
ResultContext ctx = new BasicResultContext(rb); ResultContext ctx = new BasicResultContext(rb);

View File

@ -289,27 +289,29 @@ public class PackageManager implements Closeable {
public boolean verify(SolrPackageInstance pkg, List<String> collections) { public boolean verify(SolrPackageInstance pkg, List<String> collections) {
boolean success = true; boolean success = true;
for (Plugin plugin: pkg.plugins) { for (Plugin plugin: pkg.plugins) {
for (String collection: collections) { Command cmd = plugin.verifyCommand;
Map<String, String> collectionParameterOverrides = getPackageParams(pkg.name, collection); if (plugin.verifyCommand != null && !Strings.isNullOrEmpty(cmd.path)) {
Command cmd = plugin.verifyCommand; for (String collection: collections) {
Map<String, String> collectionParameterOverrides = getPackageParams(pkg.name, collection);
Map<String, String> systemParams = Map.of("collection", collection, "package-name", pkg.name, "package-version", pkg.version); Map<String, String> systemParams = Map.of("collection", collection, "package-name", pkg.name, "package-version", pkg.version);
String url = solrBaseUrl + PackageUtils.resolve(cmd.path, pkg.parameterDefaults, collectionParameterOverrides, systemParams); String url = solrBaseUrl + PackageUtils.resolve(cmd.path, pkg.parameterDefaults, collectionParameterOverrides, systemParams);
PackageUtils.printGreen("Executing " + url + " for collection:" + collection); PackageUtils.printGreen("Executing " + url + " for collection:" + collection);
if ("GET".equalsIgnoreCase(cmd.method)) { if ("GET".equalsIgnoreCase(cmd.method)) {
String response = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), url); String response = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), url);
PackageUtils.printGreen(response); PackageUtils.printGreen(response);
String actualValue = JsonPath.parse(response, PackageUtils.jsonPathConfiguration()) String actualValue = JsonPath.parse(response, PackageUtils.jsonPathConfiguration())
.read(PackageUtils.resolve(cmd.condition, pkg.parameterDefaults, collectionParameterOverrides, systemParams)); .read(PackageUtils.resolve(cmd.condition, pkg.parameterDefaults, collectionParameterOverrides, systemParams));
String expectedValue = PackageUtils.resolve(cmd.expected, pkg.parameterDefaults, collectionParameterOverrides, systemParams); String expectedValue = PackageUtils.resolve(cmd.expected, pkg.parameterDefaults, collectionParameterOverrides, systemParams);
PackageUtils.printGreen("Actual: "+actualValue+", expected: "+expectedValue); PackageUtils.printGreen("Actual: "+actualValue+", expected: "+expectedValue);
if (!expectedValue.equals(actualValue)) { if (!expectedValue.equals(actualValue)) {
PackageUtils.printRed("Failed to deploy plugin: " + plugin.name); PackageUtils.printRed("Failed to deploy plugin: " + plugin.name);
success = false; success = false;
}
} else {
throw new SolrException(ErrorCode.BAD_REQUEST, "Non-GET method not supported for setup commands");
} }
} else {
throw new SolrException(ErrorCode.BAD_REQUEST, "Non-GET method not supported for setup commands");
} }
} }
} }

View File

@ -171,8 +171,10 @@ public class PackageUtils {
// TODO: Should perhaps use Matchers etc. instead of this clumsy replaceAll(). // TODO: Should perhaps use Matchers etc. instead of this clumsy replaceAll().
if (str == null) return null; if (str == null) return null;
for (String param: defaults.keySet()) { if (defaults != null) {
str = str.replaceAll("\\$\\{"+param+"\\}", overrides.containsKey(param)? overrides.get(param): defaults.get(param)); for (String param: defaults.keySet()) {
str = str.replaceAll("\\$\\{"+param+"\\}", overrides.containsKey(param)? overrides.get(param): defaults.get(param));
}
} }
for (String param: overrides.keySet()) { for (String param: overrides.keySet()) {
str = str.replaceAll("\\$\\{"+param+"\\}", overrides.get(param)); str = str.replaceAll("\\$\\{"+param+"\\}", overrides.get(param));

View File

@ -507,7 +507,7 @@ public final class HttpServer2 implements FilterContainer {
httpConfig.addCustomizer(new SecureRequestCustomizer()); httpConfig.addCustomizer(new SecureRequestCustomizer());
ServerConnector conn = createHttpChannelConnector(server, httpConfig); ServerConnector conn = createHttpChannelConnector(server, httpConfig);
SslContextFactory sslContextFactory = new SslContextFactory(); SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
sslContextFactory.setNeedClientAuth(needsClientAuth); sslContextFactory.setNeedClientAuth(needsClientAuth);
sslContextFactory.setKeyManagerPassword(keyPassword); sslContextFactory.setKeyManagerPassword(keyPassword);
if (keyStore != null) { if (keyStore != null) {

View File

@ -124,6 +124,7 @@ public class SystemCollectionCompatTest extends SolrCloudTestCase {
@After @After
public void doAfter() throws Exception { public void doAfter() throws Exception {
log.info("doAfter: deleting all collections...");
cluster.deleteAllCollections(); cluster.deleteAllCollections();
if (null != solrClient) { if (null != solrClient) {
@ -149,14 +150,13 @@ public class SystemCollectionCompatTest extends SolrCloudTestCase {
CollectionAdminResponse adminResponse = status.process(solrClient); CollectionAdminResponse adminResponse = status.process(solrClient);
NamedList<Object> response = adminResponse.getResponse(); NamedList<Object> response = adminResponse.getResponse();
String leader = (String) response.get("leader"); String leader = (String) response.get("leader");
log.info("Overseer Status indicates that the overseer is: {}");
JettySolrRunner overseerNode = null; JettySolrRunner overseerNode = null;
int index = -1;
List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners(); List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
for (int i = 0; i < jettySolrRunners.size(); i++) { for (int i = 0; i < jettySolrRunners.size(); i++) {
JettySolrRunner runner = jettySolrRunners.get(i); JettySolrRunner runner = jettySolrRunners.get(i);
if (runner.getNodeName().equals(leader)) { if (runner.getNodeName().equals(leader)) {
overseerNode = runner; overseerNode = runner;
index = i;
break; break;
} }
} }
@ -167,13 +167,16 @@ public class SystemCollectionCompatTest extends SolrCloudTestCase {
watcher.reset(); watcher.reset();
// restart Overseer to trigger the back-compat check // restart Overseer to trigger the back-compat check
cluster.stopJettySolrRunner(index); log.info("Stopping Overseer Node: {} ({})", overseerNode.getNodeName(), overseerNode.getLocalPort());
cluster.stopJettySolrRunner(overseerNode);
log.info("Waiting for new overseer election...");
TimeOut timeOut = new TimeOut(30, TimeUnit.SECONDS, cloudManager.getTimeSource()); TimeOut timeOut = new TimeOut(30, TimeUnit.SECONDS, cloudManager.getTimeSource());
while (!timeOut.hasTimedOut()) { while (!timeOut.hasTimedOut()) {
adminResponse = status.process(solrClient); adminResponse = status.process(solrClient);
response = adminResponse.getResponse(); response = adminResponse.getResponse();
String newLeader = (String) response.get("leader"); String newLeader = (String) response.get("leader");
if (newLeader != null && !leader.equals(newLeader)) { if (newLeader != null && !leader.equals(newLeader)) {
log.info("...new overseer is: {}", newLeader);
break; break;
} }
timeOut.sleep(200); timeOut.sleep(200);
@ -185,6 +188,9 @@ public class SystemCollectionCompatTest extends SolrCloudTestCase {
TimeOut timeOut1 = new TimeOut(60, TimeUnit.SECONDS, cloudManager.getTimeSource()); TimeOut timeOut1 = new TimeOut(60, TimeUnit.SECONDS, cloudManager.getTimeSource());
boolean foundWarning = false; boolean foundWarning = false;
boolean foundSchemaWarning = false; boolean foundSchemaWarning = false;
// TODO: replace this polling logic with a LogWatcher that uses a queue we can await() on...
log.info("Polling for log watcher to detect expected log messages...");
while (!timeOut1.hasTimedOut()) { while (!timeOut1.hasTimedOut()) {
timeOut1.sleep(1000); timeOut1.sleep(1000);
SolrDocumentList history = watcher.getHistory(-1, null); SolrDocumentList history = watcher.getHistory(-1, null);
@ -193,9 +199,11 @@ public class SystemCollectionCompatTest extends SolrCloudTestCase {
continue; continue;
} }
if (doc.getFieldValue("message").toString().contains("re-indexing")) { if (doc.getFieldValue("message").toString().contains("re-indexing")) {
log.info("Found re-indexing message: {}", doc.getFieldValue("message"));
foundWarning = true; foundWarning = true;
} }
if (doc.getFieldValue("message").toString().contains("timestamp")) { if (doc.getFieldValue("message").toString().contains("timestamp")) {
log.info("Found timestamp message: {}", doc.getFieldValue("message"));
foundSchemaWarning = true; foundSchemaWarning = true;
} }
} }
@ -203,9 +211,9 @@ public class SystemCollectionCompatTest extends SolrCloudTestCase {
break; break;
} }
} }
log.info("Done polling log watcher: foundWarning={} foundSchemaWarning={}", foundWarning, foundSchemaWarning);
assertTrue("re-indexing warning not found", foundWarning); assertTrue("re-indexing warning not found", foundWarning);
assertTrue("timestamp field incompatibility warning not found", foundSchemaWarning); assertTrue("timestamp field incompatibility warning not found", foundSchemaWarning);
} }
} }

View File

@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search;
import java.io.IOException;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrInputDocument;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@LuceneTestCase.Slow
public class FuzzySearchTest extends SolrCloudTestCase {
private final static String COLLECTION = "c1";
private CloudSolrClient client;
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(1).addConfig(COLLECTION, configset("cloud-minimal")).configure();
}
@Before
public void setupCollection() throws Exception {
client = cluster.getSolrClient();
client.setDefaultCollection(COLLECTION);
CollectionAdminRequest.createCollection(COLLECTION, 1, 1).process(client);
cluster.waitForActiveCollection(COLLECTION, 1, 1);
}
@Test
public void testTooComplex() throws IOException, SolrServerException {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("id", "1");
doc.setField("text", "foo");
client.add(doc);
client.commit(); // Must have index files written, but the contents don't matter
SolrQuery query = new SolrQuery("text:headquarters\\(在日米海軍横須賀基地司令部庁舎\\/旧横須賀鎮守府会議所・横須賀海軍艦船部\\)~");
BaseHttpSolrClient.RemoteSolrException e = expectThrows(BaseHttpSolrClient.RemoteSolrException.class, () -> client.query(query));
assertTrue("Should be client error, not server error", e.code() >= 400 && e.code() < 500);
}
}

View File

@ -41,11 +41,10 @@
<Set name="host"><Property name="jetty.host" /></Set> <Set name="host"><Property name="jetty.host" /></Set>
<Set name="port"><Property name="jetty.port" default="8983" /></Set> <Set name="port"><Property name="jetty.port" default="8983" /></Set>
<Set name="idleTimeout"><Property name="solr.jetty.http.idleTimeout" default="120000"/></Set> <Set name="idleTimeout"><Property name="solr.jetty.http.idleTimeout" default="120000"/></Set>
<Set name="soLingerTime"><Property name="solr.jetty.http.soLingerTime" default="-1"/></Set>
<Set name="acceptorPriorityDelta"><Property name="solr.jetty.http.acceptorPriorityDelta" default="0"/></Set> <Set name="acceptorPriorityDelta"><Property name="solr.jetty.http.acceptorPriorityDelta" default="0"/></Set>
<Set name="acceptQueueSize"><Property name="solr.jetty.http.acceptQueueSize" default="0"/></Set> <Set name="acceptQueueSize"><Property name="solr.jetty.http.acceptQueueSize" default="0"/></Set>
</New> </New>
</Arg> </Arg>
</Call> </Call>
</Configure> </Configure>

View File

@ -12,7 +12,7 @@
<Set name="CipherComparator"> <Set name="CipherComparator">
<Get class="org.eclipse.jetty.http2.HTTP2Cipher" name="COMPARATOR"/> <Get class="org.eclipse.jetty.http2.HTTP2Cipher" name="COMPARATOR"/>
</Set> </Set>
<Set name="useCipherSuitesOrder">true</Set> <Set name="useCipherSuitesOrder">true</Set>
</Ref> </Ref>
<!-- =========================================================== --> <!-- =========================================================== -->
@ -41,14 +41,14 @@
</New> </New>
</Item> </Item>
<Item> <Item>
<New id="alpn" class="org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory"> <New id="alpn" class="org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory">
<Arg name="protocols"> <Arg name="protocols">
<Array type="java.lang.String"> <Array type="java.lang.String">
<Item>h2</Item> <Item>h2</Item>
<Item>http/1.1</Item> <Item>http/1.1</Item>
</Array> </Array>
</Arg> </Arg>
<Set name="defaultProtocol">http/1.1</Set> <Set name="defaultProtocol">http/1.1</Set>
</New> </New>
</Item> </Item>
<Item> <Item>
@ -66,7 +66,6 @@
<Set name="host"><Property name="solr.jetty.host" /></Set> <Set name="host"><Property name="solr.jetty.host" /></Set>
<Set name="port"><Property name="solr.jetty.https.port" default="8983" /></Set> <Set name="port"><Property name="solr.jetty.https.port" default="8983" /></Set>
<Set name="idleTimeout"><Property name="solr.jetty.https.timeout" default="120000"/></Set> <Set name="idleTimeout"><Property name="solr.jetty.https.timeout" default="120000"/></Set>
<Set name="soLingerTime"><Property name="solr.jetty.https.soLingerTime" default="-1"/></Set>
<Set name="acceptorPriorityDelta"><Property name="solr.jetty.ssl.acceptorPriorityDelta" default="0"/></Set> <Set name="acceptorPriorityDelta"><Property name="solr.jetty.ssl.acceptorPriorityDelta" default="0"/></Set>
<Set name="acceptQueueSize"><Property name="solr.jetty.https.acceptQueueSize" default="0"/></Set> <Set name="acceptQueueSize"><Property name="solr.jetty.https.acceptQueueSize" default="0"/></Set>
</New> </New>

View File

@ -60,10 +60,9 @@
<Set name="host"><Property name="solr.jetty.host" /></Set> <Set name="host"><Property name="solr.jetty.host" /></Set>
<Set name="port"><Property name="solr.jetty.https.port" default="8983" /></Set> <Set name="port"><Property name="solr.jetty.https.port" default="8983" /></Set>
<Set name="idleTimeout"><Property name="solr.jetty.https.timeout" default="120000"/></Set> <Set name="idleTimeout"><Property name="solr.jetty.https.timeout" default="120000"/></Set>
<Set name="soLingerTime"><Property name="solr.jetty.https.soLingerTime" default="-1"/></Set>
<Set name="acceptorPriorityDelta"><Property name="solr.jetty.ssl.acceptorPriorityDelta" default="0"/></Set> <Set name="acceptorPriorityDelta"><Property name="solr.jetty.ssl.acceptorPriorityDelta" default="0"/></Set>
<Set name="acceptQueueSize"><Property name="solr.jetty.https.acceptQueueSize" default="0"/></Set> <Set name="acceptQueueSize"><Property name="solr.jetty.https.acceptQueueSize" default="0"/></Set>
</New> </New>
</Arg> </Arg>
</Call> </Call>
</Configure> </Configure>

View File

@ -6,7 +6,7 @@
<!-- This configuration must be used in conjunction with jetty.xml --> <!-- This configuration must be used in conjunction with jetty.xml -->
<!-- and either jetty-https.xml or jetty-spdy.xml (but not both) --> <!-- and either jetty-https.xml or jetty-spdy.xml (but not both) -->
<!-- ============================================================= --> <!-- ============================================================= -->
<Configure id="sslContextFactory" class="org.eclipse.jetty.util.ssl.SslContextFactory"> <Configure id="sslContextFactory" class="org.eclipse.jetty.util.ssl.SslContextFactory$Server">
<Call class="org.apache.solr.util.configuration.SSLConfigurationsFactory" name="current"> <Call class="org.apache.solr.util.configuration.SSLConfigurationsFactory" name="current">
<Get name="keyStorePassword" id="keyStorePassword"/> <Get name="keyStorePassword" id="keyStorePassword"/>
<Get name="trustStorePassword" id="trustStorePassword"/> <Get name="trustStorePassword" id="trustStorePassword"/>

View File

@ -132,7 +132,7 @@
<Arg> <Arg>
<New class="org.eclipse.jetty.rewrite.handler.RedirectRegexRule"> <New class="org.eclipse.jetty.rewrite.handler.RedirectRegexRule">
<Set name="regex">^/$</Set> <Set name="regex">^/$</Set>
<Set name="replacement">/solr/</Set> <Set name="location">/solr/</Set>
</New> </New>
</Arg> </Arg>
</Call> </Call>

View File

@ -1,5 +1,6 @@
= Package Management = Package Management
:page-children: package-manager-internals :page-children: package-manager-internals
:page-tocclass: right
// Licensed to the Apache Software Foundation (ASF) under one // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file // or more contributor license agreements. See the NOTICE file
@ -18,52 +19,53 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
== Glossary of Terms The package manager in Solr allows installation and update of Solr-specific packages in distributed and standalone environments.
=== Package In this system, a _package_ is a set of Java jar files (usually one) containing one or more <<solr-plugins.adoc#solr-plugins,Solr plugins>>. Each jar file is also accompanied by a signature string (which can be verified against a supplied public key).
A set of jar files (usually one) containing one or more <<solr-plugins.adoc#solr-plugins,Solr plugins>>. Each jar file is also accompanied by a signature string (which can be verified against a supplied public key).
=== Repository A key design aspect of this system is the ability to install or update packages in a cluster environment securely without the need to restart every node.
A location hosting one or many packages. Usually, this is a web service that serves meta information about packages as well as serves the package artifacts for downloading.
== Overview Other elements of the design include the ability to install from a remote repository; package standardization; a command line interface (CLI); and a package store.
The package manager in Solr consists of the following internal components:
* Package Manager CLI This section will focus on how to use the package manager to install and update plugins.
* Package Manager internal APIs For technical details, see the section <<package-manager-internals.adoc#package-manager-internals,Package Manager internals>>.
* Isolated classloaders
* Package Store
In this guide, we will focus on the Package Manager CLI, which essentially uses the other APIs and components internally. For details on the other components (and hence details of inner workings of the package manager), please refer to <<package-manager-internals.adoc#package-manager-internals,Package Manager internals>>.
== Interacting with the Package Manager == Interacting with the Package Manager
Essentially, these are the phases in using the package manager: The package manager CLI includes allows you to:
* Starting Solr with support for package management * Start Solr with support for package management
* Adding trusted repositories * Add trusted repositories
* Listing and installing packages * List packages at a repository
* Deploying packages on to collections * Install desired packages
* Updating packages * Deploy packages to collections
* Update packages when updates are available
=== Starting Solr with Package Management Support === Enable the Package Manager
Start all Solr nodes with the `-Denable.packages=true` parameter. There are security consequences in doing so. At a minimum, no unauthorized user should have write access to ZooKeeper instances, since it would then be possible to install packages from untrusted sources (e.g. malicious repositories). The package manager is disabled by default. To enable it, start all Solr nodes with the `-Denable.packages=true` parameter.
[source,bash] [source,bash]
---- ----
$ bin/solr -c -Denable.packages=true $ bin/solr -c -Denable.packages=true
---- ----
=== Adding Trusted Repositories WARNING: There are security consequences to enabling the package manager.
If an unauthorized user gained access to the system, they would have write access to ZooKeeper and could install packages from untrusted sources. Always ensure you have secured Solr with firewalls and <<authentication-and-authorization-plugins.adoc#authentication-and-authorization-plugins,authentication>> before enabling the package manager.
In order to install packages into Solr, one has to add a repository hosting the packages. A repository is essentially a web service hosting package artifacts (jar files) and a public key (to validate the signatures of the jar files while installing). Note: Please do not add repositories that you don't trust or control. Also, only add repositories that are based on https and avoid repositories based on http to safeguard against MITM attacks. === Add Trusted Repositories
A _repository_ is a a location hosting one or many packages. Often this is a web service that serves meta-information about packages, the package artifacts for downloading, and a public key to validate the jar file signatures while installing.
In order to install packages into Solr, one has to add a repository hosting the packages.
[source,bash] [source,bash]
---- ----
$ bin/solr package add-repo <name-of-repo> <repo-url> $ bin/solr package add-repo <name-of-repo> <repo-url>
---- ----
NOTE: Do not add repositories that you don't trust or control. Only add repositories that are based on HTTPS and avoid repositories based on HTTP to safeguard against MITM attacks.
=== Listing and Installing Packages === Listing and Installing Packages
To list installed packages: To list installed packages:
@ -73,7 +75,6 @@ To list installed packages:
$ bin/solr package list-installed $ bin/solr package list-installed
---- ----
To list packages available for installation from added repositories: To list packages available for installation from added repositories:
[source,bash] [source,bash]
@ -88,31 +89,38 @@ To install a package:
$ bin/solr package install <package-name>[:<version>] $ bin/solr package install <package-name>[:<version>]
---- ----
=== Deploying a Package to a Collection === Deploy a Package
Once a package has been installed, the plugins contained in it can be used in a collection, using either of the two methods: Once a package has been installed, the plugins contained in it can be used in a collection.
==== Deploying using deploy Command There are two ways to do this: either use the CLI's `deploy` command or manually.
This can be done using the package manager's `deploy` command, provided the package supports it (package author's documentation would usually mention that):
==== deploy Command
If the package author states support for it, the package can be deployed with the CLI's `deploy` command:
[source,bash] [source,bash]
---- ----
$ bin/solr package deploy <package-name>:[version] -collections <collection1>[,<collection2>,...] $ bin/solr package deploy <package-name>:[version] -collections <collection1>[,<collection2>,...]
---- ----
This may prompt you to execute a command to deploy the package. If you pass `-y` to the command, then this prompt can be skipped. The author may want you to confirm deployment of a package via a prompt.
If you pass `-y` to the command, confirmation can be skipped.
==== Deploying Manually ==== Manual Deploy
Alternatively, it is also possible manually edit a configset (solrconfig.xml, managedschema / schema.xml etc.) and using it by RELOADing a collection.
Example: Add a request handler from the package `mypackage` to a configset's solrconfig.xml: It is also possible to deploy a package manually by editing a configset (e.g., `solrconfig.xml`, `managed-schema`/`schema.xml`, etc.) and reloading the collection.
For example, if a package named `mypackage` contains a request handler, we would add it to a configset's `solrconfig.xml` like this:
[source, xml] [source, xml]
---- ----
<requestHandler name="/myhandler" class="mypackage:full.path.to.MyClass"></requestHandler> <requestHandler name="/myhandler" class="mypackage:full.path.to.MyClass"></requestHandler>
---- ----
After that, `RELOAD` your collection. Now, you should set the package version that this collection is using, as follows (say collection is called `collection1` and package name is `mypackage` and installed version is `1.0.0`): Then use either the Collections API <<collection-management.adoc#reload,RELOAD command>> or the <<collections-core-admin.adoc#collections-core-admin,Admin UI>> to reload the collection.
Next set the package version that this collection is using. If the collection is named `collection1`, the package name is `mypackage`, and the installed version is `1.0.0`, the command would look like this:
[source,bash] [source,bash]
---- ----
@ -120,26 +128,26 @@ curl "http://localhost:8983/api/collections/collection1/config/params" \
-H 'Content-type:application/json' -d "{set: {PKG_VERSIONS: {mypackage: '1.0.0'}}}" -H 'Content-type:application/json' -d "{set: {PKG_VERSIONS: {mypackage: '1.0.0'}}}"
---- ----
==== Verifying the Deployment ==== Verify the Deployment
After deploying, verify that the collection is using the package: After deploying, verify that the collection is using the package:
[source,bash] [source,bash]
---- ----
$ bin/solr package list-deployed -c <collection> $ bin/solr package list-deployed -c <collection>
---- ----
=== Updating Packages === Updating Packages
In order to update a package, first step is make sure the updated version is available in the added repositories by running `list-available` command. Next, install the new version of the package from the repositories. In order to update a package, first step is make sure the updated version is available in the added repositories by running `list-available` command shown above in <<Listing and Installing Packages>>.
Next, install the new version of the package from the repositories.
[source,bash] [source,bash]
---- ----
$ bin/solr package install <package-name>:<version> $ bin/solr package install <package-name>:<version>
---- ----
Now, you can selectively update each of your collections using the old version (say, `1.0.0`) of the package (say, `mypackage`) to the newly added version (say `2.0.0`) as follows: Once you have installed the new version, you can selectively update each of your collections. Assuming the old version is `1.0.0` of the package `mypackage`, and the new version is `2.0.0`, the command would be as follows:
[source,bash] [source,bash]
---- ----
@ -149,6 +157,7 @@ $ bin/solr package deploy mypackage:2.0.0 --update -collections mycollection
You can run the `list-deployed` command to verify that this collection is using the newly added version. You can run the `list-deployed` command to verify that this collection is using the newly added version.
== Security == Security
Except the `add-repo` step, all other steps can be executed using a HTTP endpoint in Solr (see <<package-manager-internals.adoc#package-manager-internals,Package Manager internals>>). This step registers the public key of the trusted repository, and hence can only be executed using the package manager (CLI) having direct write access to ZooKeeper. Hence, as you can imagine, it is important to protect ZooKeeper from unauthorized write access.
Also, keep in mind, that it is possible to install any package from a trusted and an already added repository. Hence, if you want to use some packages in production, then it is better to setup your own repository and add that to Solr, instead of adding a generic third-party repository that is beyond your administrative control. As noted above in the section <<Add Trusted Repositories>>, the `add-repo` step should only be executed using an HTTPS endpoint in Solr (all other steps can be executed using HTTP - see also <<package-manager-internals.adoc#package-manager-internals,Package Manager Internals>>). This step registers the public key of the trusted repository, and hence can only be executed using the package manager (CLI) having direct write access to ZooKeeper. It is critical to protect ZooKeeper from unauthorized write access.
Also, keep in mind, that it is possible to install *any* package from a repository once it has been added. If you want to use some packages in production, a best practice is to setup your own repository and add that to Solr instead of adding a generic third-party repository that is beyond your administrative control.

View File

@ -24,7 +24,6 @@ import org.eclipse.jetty.util.ssl.SslContextFactory;
* @see #setUseSSL * @see #setUseSSL
*/ */
public class SSLConfig { public class SSLConfig {
private boolean useSsl; private boolean useSsl;
private boolean clientAuth; private boolean clientAuth;
private String keyStore; private String keyStore;
@ -76,7 +75,7 @@ public class SSLConfig {
} }
/** /**
* Returns an SslContextFactory that should be used by a jetty server based on the specified * Returns an SslContextFactory.Server that should be used by a jetty server based on the specified
* SSLConfig param which may be null. * SSLConfig param which may be null.
* *
* if the SSLConfig param is non-null, then this method will return the results of * if the SSLConfig param is non-null, then this method will return the results of
@ -88,8 +87,7 @@ public class SSLConfig {
* *
* @see #createContextFactory() * @see #createContextFactory()
*/ */
public static SslContextFactory createContextFactory(SSLConfig sslConfig) { public static SslContextFactory.Server createContextFactory(SSLConfig sslConfig) {
if (sslConfig != null) { if (sslConfig != null) {
return sslConfig.createContextFactory(); return sslConfig.createContextFactory();
} }
@ -102,7 +100,7 @@ public class SSLConfig {
} }
/** /**
* Returns an SslContextFactory that should be used by a jetty server based on this SSLConfig instance, * Returns an SslContextFactory.Server that should be used by a jetty server based on this SSLConfig instance,
* or null if SSL should not be used. * or null if SSL should not be used.
* *
* The default implementation generates a simple factory according to the keystore, truststore, * The default implementation generates a simple factory according to the keystore, truststore,
@ -114,14 +112,13 @@ public class SSLConfig {
* @see #getTrustStore * @see #getTrustStore
* @see #getTrustStorePassword * @see #getTrustStorePassword
*/ */
public SslContextFactory createContextFactory() { public SslContextFactory.Server createContextFactory() {
if (! isSSLMode()) { if (! isSSLMode()) {
return null; return null;
} }
// else... // else...
SslContextFactory factory = new SslContextFactory(false); SslContextFactory.Server factory = new SslContextFactory.Server();
if (getKeyStore() != null) if (getKeyStore() != null)
factory.setKeyStorePath(getKeyStore()); factory.setKeyStorePath(getKeyStore());
if (getKeyStorePassword() != null) if (getKeyStorePassword() != null)
@ -136,12 +133,14 @@ public class SSLConfig {
factory.setTrustStorePassword(getTrustStorePassword()); factory.setTrustStorePassword(getTrustStorePassword());
} }
return factory; return factory;
} }
private static SslContextFactory configureSslFromSysProps() { public SslContextFactory.Client createClientContextFactory() {
return new SslContextFactory.Client();
}
SslContextFactory sslcontext = new SslContextFactory(false); private static SslContextFactory.Server configureSslFromSysProps() {
SslContextFactory.Server sslcontext = new SslContextFactory.Server();
if (null != System.getProperty("javax.net.ssl.keyStore")) { if (null != System.getProperty("javax.net.ssl.keyStore")) {
sslcontext.setKeyStorePath sslcontext.setKeyStorePath

View File

@ -99,7 +99,6 @@ import org.slf4j.LoggerFactory;
import static org.apache.solr.client.solrj.impl.BaseHttpSolrClient.*; import static org.apache.solr.client.solrj.impl.BaseHttpSolrClient.*;
import static org.apache.solr.common.util.Utils.getObjectByPath; import static org.apache.solr.common.util.Utils.getObjectByPath;
// TODO: error handling, small Http2SolrClient features, security, ssl
/** /**
* Difference between this {@link Http2SolrClient} and {@link HttpSolrClient}: * Difference between this {@link Http2SolrClient} and {@link HttpSolrClient}:
* <ul> * <ul>
@ -180,13 +179,13 @@ public class Http2SolrClient extends SolrClient {
ThreadPoolExecutor httpClientExecutor = new ExecutorUtil.MDCAwareThreadPoolExecutor(32, ThreadPoolExecutor httpClientExecutor = new ExecutorUtil.MDCAwareThreadPoolExecutor(32,
256, 60, TimeUnit.SECONDS, queue, new SolrjNamedThreadFactory("h2sc")); 256, 60, TimeUnit.SECONDS, queue, new SolrjNamedThreadFactory("h2sc"));
SslContextFactory sslContextFactory; SslContextFactory.Client sslContextFactory;
boolean ssl; boolean ssl;
if (builder.sslConfig == null) { if (builder.sslConfig == null) {
sslContextFactory = getDefaultSslContextFactory(); sslContextFactory = getDefaultSslContextFactory();
ssl = sslContextFactory.getTrustStore() != null || sslContextFactory.getTrustStorePath() != null; ssl = sslContextFactory.getTrustStore() != null || sslContextFactory.getTrustStorePath() != null;
} else { } else {
sslContextFactory = builder.sslConfig.createContextFactory(); sslContextFactory = builder.sslConfig.createClientContextFactory();
ssl = true; ssl = true;
} }
@ -868,7 +867,6 @@ public class Http2SolrClient extends SolrClient {
this.connectionTimeout = connectionTimeOut; this.connectionTimeout = connectionTimeOut;
return this; return this;
} }
} }
public Set<String> getQueryParams() { public Set<String> getQueryParams() {
@ -921,7 +919,7 @@ public class Http2SolrClient extends SolrClient {
Http2SolrClient.defaultSSLConfig = null; Http2SolrClient.defaultSSLConfig = null;
} }
private static SslContextFactory getDefaultSslContextFactory() { private static SslContextFactory.Client getDefaultSslContextFactory() {
String checkPeerNameStr = System.getProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME); String checkPeerNameStr = System.getProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME);
boolean sslCheckPeerName = true; boolean sslCheckPeerName = true;
if (checkPeerNameStr == null || "false".equalsIgnoreCase(checkPeerNameStr)) { if (checkPeerNameStr == null || "false".equalsIgnoreCase(checkPeerNameStr)) {
@ -949,5 +947,4 @@ public class Http2SolrClient extends SolrClient {
return sslContextFactory; return sslContextFactory;
} }
} }

View File

@ -43,7 +43,7 @@ public class ConcatEvaluator extends RecursiveObjectEvaluator implements ManyVal
} }
@Override @Override
public Object doWork(Object values[]) throws IOException { public Object doWork(Object... values) throws IOException {
StringBuilder buff = new StringBuilder(); StringBuilder buff = new StringBuilder();

View File

@ -41,7 +41,7 @@ public class DateEvaluator extends RecursiveObjectEvaluator implements ManyValue
} }
@Override @Override
public Object doWork(Object values[]) throws IOException { public Object doWork(Object... values) throws IOException {
String sdate = values[0].toString(); String sdate = values[0].toString();
String template = values[1].toString(); String template = values[1].toString();

View File

@ -41,7 +41,7 @@ public class DbscanEvaluator extends RecursiveObjectEvaluator implements ManyVal
} }
@Override @Override
public Object doWork(Object values[]) throws IOException { public Object doWork(Object... values) throws IOException {
Matrix matrix = null; Matrix matrix = null;
double e = 0; double e = 0;

View File

@ -38,7 +38,7 @@ public class EmpiricalDistributionEvaluator extends RecursiveNumericEvaluator im
} }
@Override @Override
public Object doWork(Object[] values) throws IOException { public Object doWork(Object... values) throws IOException {
if(!(values[0] instanceof List<?>)){ if(!(values[0] instanceof List<?>)){
throw new StreamEvaluatorException("List value expected but found type %s for value %s", values[0].getClass().getName(), values[0].toString()); throw new StreamEvaluatorException("List value expected but found type %s for value %s", values[0].getClass().getName(), values[0].toString());

View File

@ -36,7 +36,7 @@ public class MatchesEvaluator extends RecursiveBooleanEvaluator implements ManyV
} }
} }
public Object doWork(Object[] values) throws IOException { public Object doWork(Object... values) throws IOException {
if(values[1] instanceof String) { if(values[1] instanceof String) {
String s = values[0].toString(); String s = values[0].toString();
if(pattern == null) { if(pattern == null) {

View File

@ -42,7 +42,7 @@ public class PivotEvaluator extends RecursiveObjectEvaluator implements ManyValu
} }
@Override @Override
public Object doWork(Object[] values) throws IOException { public Object doWork(Object... values) throws IOException {
if(values.length != 4) { if(values.length != 4) {
throw new IOException("The pivot function requires four parameters."); throw new IOException("The pivot function requires four parameters.");
} }

View File

@ -25,9 +25,9 @@ import org.eclipse.jetty.webapp.WebAppContext;
/** /**
* @since solr 1.3 * @since solr 1.3
*/ */
public class StartSolrJetty public class StartSolrJetty
{ {
public static void main( String[] args ) public static void main( String[] args )
{ {
//System.setProperty("solr.solr.home", "../../../example/solr"); //System.setProperty("solr.solr.home", "../../../example/solr");
@ -35,10 +35,9 @@ public class StartSolrJetty
ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory()); ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory());
// Set some timeout options to make debugging easier. // Set some timeout options to make debugging easier.
connector.setIdleTimeout(1000 * 60 * 60); connector.setIdleTimeout(1000 * 60 * 60);
connector.setSoLingerTime(-1);
connector.setPort(8983); connector.setPort(8983);
server.setConnectors(new Connector[] { connector }); server.setConnectors(new Connector[] { connector });
WebAppContext bb = new WebAppContext(); WebAppContext bb = new WebAppContext();
bb.setServer(server); bb.setServer(server);
bb.setContextPath("/solr"); bb.setContextPath("/solr");
@ -51,7 +50,7 @@ public class StartSolrJetty
// server.getContainer().addEventListener(mBeanContainer); // server.getContainer().addEventListener(mBeanContainer);
// mBeanContainer.start(); // mBeanContainer.start();
// } // }
server.setHandler(bb); server.setHandler(bb);
try { try {
@ -62,7 +61,7 @@ public class StartSolrJetty
} }
server.stop(); server.stop();
server.join(); server.join();
} }
catch (Exception e) { catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
System.exit(100); System.exit(100);

View File

@ -47,13 +47,13 @@ import org.junit.rules.TestRule;
* *
* @since solr 1.3 * @since solr 1.3
*/ */
public class JettyWebappTest extends SolrTestCaseJ4 public class JettyWebappTest extends SolrTestCaseJ4
{ {
int port = 0; int port = 0;
static final String context = "/test"; static final String context = "/test";
@Rule @Rule
public TestRule solrTestRules = public TestRule solrTestRules =
RuleChain.outerRule(new SystemPropertiesRestoreRule()); RuleChain.outerRule(new SystemPropertiesRestoreRule());
Server server; Server server;
@ -65,7 +65,7 @@ public class JettyWebappTest extends SolrTestCaseJ4
System.setProperty("solr.solr.home", SolrJettyTestBase.legacyExampleCollection1SolrHome()); System.setProperty("solr.solr.home", SolrJettyTestBase.legacyExampleCollection1SolrHome());
System.setProperty("tests.shardhandler.randomSeed", Long.toString(random().nextLong())); System.setProperty("tests.shardhandler.randomSeed", Long.toString(random().nextLong()));
System.setProperty("solr.tests.doContainerStreamCloseAssert", "false"); System.setProperty("solr.tests.doContainerStreamCloseAssert", "false");
File dataDir = createTempDir().toFile(); File dataDir = createTempDir().toFile();
dataDir.mkdirs(); dataDir.mkdirs();
@ -79,11 +79,10 @@ public class JettyWebappTest extends SolrTestCaseJ4
ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory()); ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory());
connector.setIdleTimeout(1000 * 60 * 60); connector.setIdleTimeout(1000 * 60 * 60);
connector.setSoLingerTime(-1);
connector.setPort(0); connector.setPort(0);
server.setConnectors(new Connector[]{connector}); server.setConnectors(new Connector[]{connector});
server.setStopAtShutdown( true ); server.setStopAtShutdown( true );
server.start(); server.start();
port = connector.getLocalPort(); port = connector.getLocalPort();
} }
@ -99,12 +98,12 @@ public class JettyWebappTest extends SolrTestCaseJ4
System.clearProperty("solr.tests.doContainerStreamCloseAssert"); System.clearProperty("solr.tests.doContainerStreamCloseAssert");
super.tearDown(); super.tearDown();
} }
public void testAdminUI() throws Exception public void testAdminUI() throws Exception
{ {
// Currently not an extensive test, but it does fire up the JSP pages and make // Currently not an extensive test, but it does fire up the JSP pages and make
// sure they compile ok // sure they compile ok
String adminPath = "http://127.0.0.1:"+port+context+"/"; String adminPath = "http://127.0.0.1:"+port+context+"/";
byte[] bytes = IOUtils.toByteArray( new URL(adminPath).openStream() ); byte[] bytes = IOUtils.toByteArray( new URL(adminPath).openStream() );
assertNotNull( bytes ); // real error will be an exception assertNotNull( bytes ); // real error will be an exception

View File

@ -184,7 +184,7 @@ public class SSLTestConfig {
return new SSLConfig(isSSLMode(), isClientAuthMode(), null, null, null, null) { return new SSLConfig(isSSLMode(), isClientAuthMode(), null, null, null, null) {
@Override @Override
public SslContextFactory createContextFactory() { public SslContextFactory.Client createClientContextFactory() {
SslContextFactory.Client factory = new SslContextFactory.Client(!checkPeerName); SslContextFactory.Client factory = new SslContextFactory.Client(!checkPeerName);
try { try {
factory.setSslContext(buildClientSSLContext()); factory.setSslContext(buildClientSSLContext());
@ -212,7 +212,7 @@ public class SSLTestConfig {
return new SSLConfig(isSSLMode(), isClientAuthMode(), null, null, null, null) { return new SSLConfig(isSSLMode(), isClientAuthMode(), null, null, null, null) {
@Override @Override
public SslContextFactory createContextFactory() { public SslContextFactory.Server createContextFactory() {
SslContextFactory.Server factory = new SslContextFactory.Server(); SslContextFactory.Server factory = new SslContextFactory.Server();
try { try {
SSLContextBuilder builder = SSLContexts.custom(); SSLContextBuilder builder = SSLContexts.custom();

View File

@ -1,95 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.util;
import java.security.AccessController;
import java.security.PrivilegedAction;
/**
* A {@link SecurityManager} that prevents tests calling {@link System#exit(int)}.
* Only the test runner itself is allowed to exit the JVM.
* All other security checks are handled by the default security policy.
* <p>
* Use this with {@code -Djava.security.manager=org.apache.solr.util.SolrSecurityManager}.
*/
public final class SolrSecurityManager extends SecurityManager {
static final String JUNIT4_TEST_RUNNER_PACKAGE = "com.carrotsearch.ant.tasks.junit4.";
static final String ECLIPSE_TEST_RUNNER_PACKAGE = "org.eclipse.jdt.internal.junit.runner.";
static final String IDEA_TEST_RUNNER_PACKAGE = "com.intellij.rt.execution.junit.";
static final String GRADLE_TEST_RUNNER_PACKAGE = "worker.org.gradle.process.internal.worker";
/**
* Creates a new SolrSecurityManager. This ctor is called on JVM startup,
* when {@code -Djava.security.manager=org.apache.lucene.util.TestSecurityManager}
* is passed to JVM.
*/
public SolrSecurityManager() {
super();
}
/**
* {@inheritDoc}
* <p>This method inspects the stack trace and checks who is calling
* {@link System#exit(int)} and similar methods
* @throws SecurityException if the caller of this method is not the test runner itself.
*/
@Override
public void checkExit(final int status) {
AccessController.doPrivileged(new PrivilegedAction<Void>() {
@Override
public Void run() {
final String systemClassName = System.class.getName(),
runtimeClassName = Runtime.class.getName();
String exitMethodHit = null;
for (final StackTraceElement se : Thread.currentThread().getStackTrace()) {
final String className = se.getClassName(), methodName = se.getMethodName();
if (
("exit".equals(methodName) || "halt".equals(methodName)) &&
(systemClassName.equals(className) || runtimeClassName.equals(className))
) {
exitMethodHit = className + '#' + methodName + '(' + status + ')';
continue;
}
if (exitMethodHit != null) {
if (className.startsWith(JUNIT4_TEST_RUNNER_PACKAGE) ||
className.startsWith(ECLIPSE_TEST_RUNNER_PACKAGE) ||
className.startsWith(IDEA_TEST_RUNNER_PACKAGE) ||
className.startsWith(GRADLE_TEST_RUNNER_PACKAGE)) {
// this exit point is allowed, we return normally from closure:
return /*void*/ null;
} else {
// anything else in stack trace is not allowed, break and throw SecurityException below:
break;
}
}
}
if (exitMethodHit == null) {
// should never happen, only if JVM hides stack trace - replace by generic:
exitMethodHit = "JVM exit method";
}
throw new SecurityException(exitMethodHit + " calls are not allowed because they terminate the test runner's JVM.");
}
});
// we passed the stack check, delegate to super, so default policy can still deny permission:
super.checkExit(status);
}
}

View File

@ -34,7 +34,24 @@ public class TestLogLevelAnnotations extends SolrTestCaseJ4 {
private static final String bogus_logger_prefix = "org.apache.solr.bogus_logger"; private static final String bogus_logger_prefix = "org.apache.solr.bogus_logger";
/** /**
* We don't want a hardocded assumption here because it may change based on how the user runs the test. * <p>
* The default log level of the root logger when this class is loaded by the JVM, Used to validate
* some logger configurations when the tests are run.
* </p>
* <p>
* We don't want a hardcoded assumption here because it may change based on how the user runs the test.
* </p>
* <p>
* We also don't want to initialize this in a <code>@BeforeClass</code> method because that will run
* <em>after</em> the <code>@BeforeClass</code> logic of our super class {@link SolrTestCaseJ4}
* where the <code>@LogLevel</code> annotation on this class will be parsed and evaluated -- modifying the
* log4j run time configuration.
* There is no reason why the <code>@LogLevel</code> configuration of this class <em>should</em> affect
* the "root" Logger, but setting this in static class initialization protect us (as best we can)
* against the possibility that it <em>might</em> due to an unforseen (future) bug.
* </p>
*
* @see #checkLogLevelsBeforeClass
*/ */
public static final Level DEFAULT_LOG_LEVEL = LogManager.getRootLogger().getLevel(); public static final Level DEFAULT_LOG_LEVEL = LogManager.getRootLogger().getLevel();
@ -49,7 +66,8 @@ public class TestLogLevelAnnotations extends SolrTestCaseJ4 {
final Configuration config = ctx.getConfiguration(); final Configuration config = ctx.getConfiguration();
// NOTE: we're checking the CONFIGURATION of the loggers, not the "effective" value of the Logger // NOTE: we're checking the CONFIGURATION of the loggers, not the "effective" value of the Logger
assertEquals(DEFAULT_LOG_LEVEL, config.getRootLogger().getLevel()); assertEquals("Somehow, the configured value of the root logger changed since this class was loaded",
DEFAULT_LOG_LEVEL, config.getRootLogger().getLevel());
assertEquals("Your Logger conf sets a level on a bogus package that breaks this test: " assertEquals("Your Logger conf sets a level on a bogus package that breaks this test: "
+ bogus_logger_prefix, + bogus_logger_prefix,
config.getRootLogger(), config.getRootLogger(),

View File

@ -21,13 +21,9 @@ solrAdminApp.controller('IndexController', function($scope, System, Cores, Const
System.get(function(data) { System.get(function(data) {
$scope.system = data; $scope.system = data;
// load average // load average, unless its negative (means n/a on windows, etc)
var load_average = ( data.system.uptime || '' ).match( /load averages?: (\d+[.,]\d\d),? (\d+[.,]\d\d),? (\d+[.,]\d\d)/ ); if (data.system.systemLoadAverage >= 0) {
if (load_average) { $scope.load_average = data.system.systemLoadAverage.toFixed(2);
for (var i=0;i<2;i++) {
load_average[i]=load_average[i].replace(",","."); // for European users
}
$scope.load_average = load_average.slice(1);
} }
// physical memory // physical memory

View File

@ -106,7 +106,7 @@ limitations under the License.
<div class="block" id="system"> <div class="block" id="system">
<h2><span>System</span> <h2><span>System</span>
<small class="bar-desc">{{load_average[0]}} {{load_average[1]}} {{load_average[2]}}</small> <small class="bar-desc">{{load_average}}</small>
</h2> </h2>
<a class="reload" ng-click="reload()"><span>reload</span></a> <a class="reload" ng-click="reload()"><span>reload</span></a>