Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-07 02:28:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge jira/http2 branch to master

Browse Source
This commit is contained in:
Cao Manh Dat 2018-12-16 16:58:20 +00:00
parent dae3e304a1
commit f80e8e1167
102 changed files with 7336 additions and 1091 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lucene/ivy-versions.properties
View File

@ -245,6 +245,16 @@ org.codehaus.janino.version = 2.7.6
/org.codehaus.woodstox/woodstox-core-asl = 4.4.1
org.eclipse.jetty.version = 9.4.14.v20181114
/org.eclipse.jetty.http2/http2-client = ${org.eclipse.jetty.version}
/org.eclipse.jetty.http2/http2-common = ${org.eclipse.jetty.version}
/org.eclipse.jetty.http2/http2-hpack = ${org.eclipse.jetty.version}
/org.eclipse.jetty.http2/http2-http-client-transport = ${org.eclipse.jetty.version}
/org.eclipse.jetty.http2/http2-server = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-alpn-client = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-alpn-java-client = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-alpn-java-server = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-alpn-server = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-client = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-continuation = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-deploy = ${org.eclipse.jetty.version}
/org.eclipse.jetty/jetty-http = ${org.eclipse.jetty.version}

9
solr/bin/solr
View File

@ -176,7 +176,14 @@ if [ -z "$SOLR_SSL_ENABLED" ]; then
fi
fi
if [ "$SOLR_SSL_ENABLED" == "true" ]; then
SOLR_JETTY_CONFIG+=("--module=https")
if [[ "$JAVA_VER_NUM" -lt "9" ]] ; then
echo >&2 "HTTP/2 + SSL is not support in Java 8. "
echo >&2 "Configure Solr with HTTP/1.1 + SSL"
SOLR_JETTY_CONFIG+=("--module=https8")
else
SOLR_JETTY_CONFIG+=("--module=https")
fi
SOLR_JETTY_CONFIG+=("--lib=$DEFAULT_SERVER_DIR/solr-webapp/webapp/WEB-INF/lib/*")
SOLR_URL_SCHEME=https
if [ -n "$SOLR_SSL_KEY_STORE" ]; then

15
solr/core/src/java/org/apache/solr/client/solrj/embedded/JettyConfig.java
View File

@ -25,6 +25,9 @@ import java.util.TreeMap;
public class JettyConfig {
// by default jetty will start with http2 + http1 support
public final boolean onlyHttp1;
public final int port;
public final String context;
@ -41,8 +44,10 @@ public class JettyConfig {
public final int portRetryTime;
private JettyConfig(int port, int portRetryTime, String context, boolean stopAtShutdown, Long waitForLoadingCoresToFinishMs, Map<ServletHolder, String> extraServlets,
private JettyConfig(boolean onlyHttp1, int port, int portRetryTime , String context, boolean stopAtShutdown,
Long waitForLoadingCoresToFinishMs, Map<ServletHolder, String> extraServlets,
Map<Class<? extends Filter>, String> extraFilters, SSLConfig sslConfig) {
this.onlyHttp1 = onlyHttp1;
this.port = port;
this.context = context;
this.stopAtShutdown = stopAtShutdown;
@ -70,6 +75,7 @@ public class JettyConfig {
public static class Builder {
boolean onlyHttp1 = false;
int port = 0;
String context = "/solr";
boolean stopAtShutdown = true;
@ -79,6 +85,11 @@ public class JettyConfig {
SSLConfig sslConfig = null;
int portRetryTime = 60;
public Builder useOnlyHttp1(boolean useOnlyHttp1) {
this.onlyHttp1 = useOnlyHttp1;
return this;
}
public Builder setPort(int port) {
this.port = port;
return this;
@ -133,7 +144,7 @@ public class JettyConfig {
public JettyConfig build() {
return new JettyConfig(port, portRetryTime, context, stopAtShutdown, waitForLoadingCoresToFinishMs, extraServlets, extraFilters, sslConfig);
return new JettyConfig(onlyHttp1, port, portRetryTime, context, stopAtShutdown, waitForLoadingCoresToFinishMs, extraServlets, extraFilters, sslConfig);
}
}

48
solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
View File

@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.EnumSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
@ -44,6 +45,7 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.lucene.util.Constants;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.cloud.SocketProxy;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
@ -52,6 +54,10 @@ import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.servlet.SolrDispatchFilter;
import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory;
import org.eclipse.jetty.http2.HTTP2Cipher;
import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory;
import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory;
import org.apache.solr.util.TimeOut;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
@ -271,16 +277,43 @@ public class JettySolrRunner {
// talking to that server, but for the purposes of testing that should
// be good enough
final SslContextFactory sslcontext = SSLConfig.createContextFactory(config.sslConfig);
HttpConfiguration configuration = new HttpConfiguration();
ServerConnector connector;
if (sslcontext != null) {
HttpConfiguration configuration = new HttpConfiguration();
configuration.setSecureScheme("https");
configuration.addCustomizer(new SecureRequestCustomizer());
connector = new ServerConnector(server, new SslConnectionFactory(sslcontext, "http/1.1"),
new HttpConnectionFactory(configuration));
HttpConnectionFactory http1ConnectionFactory = new HttpConnectionFactory(configuration);
if (config.onlyHttp1 || !Constants.JRE_IS_MINIMUM_JAVA9) {
connector = new ServerConnector(server, new SslConnectionFactory(sslcontext,
http1ConnectionFactory.getProtocol()),
http1ConnectionFactory);
} else {
sslcontext.setCipherComparator(HTTP2Cipher.COMPARATOR);
connector = new ServerConnector(server);
SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslcontext, "alpn");
connector.addConnectionFactory(sslConnectionFactory);
connector.setDefaultProtocol(sslConnectionFactory.getProtocol());
HTTP2ServerConnectionFactory http2ConnectionFactory = new HTTP2ServerConnectionFactory(configuration);
ALPNServerConnectionFactory alpn = new ALPNServerConnectionFactory(
http2ConnectionFactory.getProtocol(),
http1ConnectionFactory.getProtocol());
alpn.setDefaultProtocol(http1ConnectionFactory.getProtocol());
connector.addConnectionFactory(alpn);
connector.addConnectionFactory(http1ConnectionFactory);
connector.addConnectionFactory(http2ConnectionFactory);
}
} else {
connector = new ServerConnector(server, new HttpConnectionFactory());
if (config.onlyHttp1) {
connector = new ServerConnector(server, new HttpConnectionFactory(configuration));
} else {
connector = new ServerConnector(server, new HttpConnectionFactory(configuration),
new HTTP2CServerConnectionFactory(configuration));
}
}
connector.setReuseAddress(true);
@ -292,7 +325,8 @@ public class JettySolrRunner {
server.setConnectors(new Connector[] {connector});
server.setSessionIdManager(new DefaultSessionIdManager(server, new Random()));
} else {
ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory());
HttpConfiguration configuration = new HttpConfiguration();
ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(configuration));
connector.setPort(port);
connector.setSoLingerTime(-1);
connector.setIdleTimeout(THREAD_POOL_MAX_IDLE_TIME_MS);
@ -495,7 +529,7 @@ public class JettySolrRunner {
}
ServerConnector c = (ServerConnector) conns[0];
protocol = c.getDefaultProtocol().startsWith("SSL") ? "https" : "http";
protocol = c.getDefaultProtocol().toLowerCase(Locale.ROOT).startsWith("ssl") ? "https" : "http";
this.protocol = protocol;
this.host = c.getHost();

9
solr/core/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -405,8 +405,11 @@ public class CoreContainer {
private void setupHttpClientForAuthPlugin(Object authcPlugin) {
if (authcPlugin instanceof HttpClientBuilderPlugin) {
// Setup HttpClient for internode communication
SolrHttpClientBuilder builder = ((HttpClientBuilderPlugin) authcPlugin).getHttpClientBuilder(HttpClientUtil.getHttpClientBuilder());
HttpClientBuilderPlugin builderPlugin = ((HttpClientBuilderPlugin) authcPlugin);
SolrHttpClientBuilder builder = builderPlugin.getHttpClientBuilder(HttpClientUtil.getHttpClientBuilder());
shardHandlerFactory.setSecurityBuilder(builderPlugin);
updateShardHandler.setSecurityBuilder(builderPlugin);
// The default http client of the core container's shardHandlerFactory has already been created and
// configured using the default httpclient configurer. We need to reconfigure it using the plugin's
// http client configurer to set it up for internode communication.
@ -438,6 +441,8 @@ public class CoreContainer {
// each request to the configured authentication plugin.
if (pkiAuthenticationPlugin != null && !pkiAuthenticationPlugin.isInterceptorRegistered()) {
pkiAuthenticationPlugin.getHttpClientBuilder(HttpClientUtil.getHttpClientBuilder());
shardHandlerFactory.setSecurityBuilder(pkiAuthenticationPlugin);
updateShardHandler.setSecurityBuilder(pkiAuthenticationPlugin);
}
}

23
solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.solr.handler.component;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.ConnectException;
import java.util.ArrayList;
@ -32,12 +33,11 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.LBSolrClient;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.cloud.CloudDescriptor;
@ -75,11 +75,11 @@ public class HttpShardHandler extends ShardHandler {
private CompletionService<ShardResponse> completionService;
private Set<Future<ShardResponse>> pending;
private Map<String,List<String>> shardToURLs;
private HttpClient httpClient;
private Http2SolrClient httpClient;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public HttpShardHandler(HttpShardHandlerFactory httpShardHandlerFactory, HttpClient httpClient) {
public HttpShardHandler(HttpShardHandlerFactory httpShardHandlerFactory, Http2SolrClient httpClient) {
this.httpClient = httpClient;
this.httpShardHandlerFactory = httpShardHandlerFactory;
completionService = httpShardHandlerFactory.newCompletionService();
@ -171,11 +171,9 @@ public class HttpShardHandler extends ShardHandler {
if (urls.size() <= 1) {
String url = urls.get(0);
srsp.setShardAddress(url);
try (SolrClient client = new Builder(url).withHttpClient(httpClient).build()) {
ssr.nl = client.request(req);
}
ssr.nl = request(url, req);
} else {
LBHttpSolrClient.Rsp rsp = httpShardHandlerFactory.makeLoadBalancedRequest(req, urls);
LBSolrClient.Rsp rsp = httpShardHandlerFactory.makeLoadBalancedRequest(req, urls);
ssr.nl = rsp.getResponse();
srsp.setShardAddress(rsp.getServer());
}
@ -209,6 +207,11 @@ public class HttpShardHandler extends ShardHandler {
MDC.remove("ShardRequest.urlList");
}
}
protected NamedList<Object> request(String url, SolrRequest req) throws IOException, SolrServerException {
req.setBasePath(url);
return httpClient.request(req);
}
/**
* Subclasses could modify the request based on the shard

162
solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java
View File

@ -16,39 +16,6 @@
*/
package org.apache.solr.handler.component;
import org.apache.commons.lang.StringUtils;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient.Builder;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.URLUtil;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.update.UpdateShardHandlerConfig;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.stats.HttpClientMetricNameStrategy;
import org.apache.solr.util.stats.InstrumentedHttpRequestExecutor;
import org.apache.solr.util.stats.InstrumentedPoolingHttpClientConnectionManager;
import org.apache.solr.util.stats.MetricUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
@ -64,8 +31,42 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.TimeUnit;
import static org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.KNOWN_METRIC_NAME_STRATEGIES;
import org.apache.commons.lang.StringUtils;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.LBHttp2SolrClient;
import org.apache.solr.client.solrj.impl.LBSolrClient;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.URLUtil;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.security.HttpClientBuilderPlugin;
import org.apache.solr.update.UpdateShardHandlerConfig;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.stats.InstrumentedHttpListenerFactory;
import org.apache.solr.util.stats.MetricUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.util.stats.InstrumentedHttpListenerFactory.KNOWN_METRIC_NAME_STRATEGIES;
public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.apache.solr.util.plugin.PluginInfoInitialized, SolrMetricProducer {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ -88,15 +89,10 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
false
);
protected InstrumentedPoolingHttpClientConnectionManager clientConnectionManager;
protected CloseableHttpClient defaultClient;
protected InstrumentedHttpRequestExecutor httpRequestExecutor;
private LBHttpSolrClient loadbalancer;
//default values:
int soTimeout = HttpClientUtil.DEFAULT_SO_TIMEOUT;
int connectionTimeout = HttpClientUtil.DEFAULT_CONNECT_TIMEOUT;
int maxConnectionsPerHost = HttpClientUtil.DEFAULT_MAXCONNECTIONSPERHOST;
int maxConnections = HttpClientUtil.DEFAULT_MAXCONNECTIONS;
protected volatile Http2SolrClient defaultClient;
protected InstrumentedHttpListenerFactory httpListenerFactory;
private LBHttp2SolrClient loadbalancer;
int corePoolSize = 0;
int maximumPoolSize = Integer.MAX_VALUE;
int keepAliveTime = 5;
@ -107,9 +103,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
private String scheme = null;
private HttpClientMetricNameStrategy metricNameStrategy;
private String metricTag;
private InstrumentedHttpListenerFactory.NameStrategy metricNameStrategy;
protected final Random r = new Random();
@ -150,15 +144,27 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
/**
* Get {@link ShardHandler} that uses custom http client.
*/
public ShardHandler getShardHandler(final HttpClient httpClient){
public ShardHandler getShardHandler(final Http2SolrClient httpClient){
return new HttpShardHandler(this, httpClient);
}
@Deprecated
public ShardHandler getShardHandler(final HttpClient httpClient) {
// a little hack for backward-compatibility when we are moving from apache http client to jetty client
return new HttpShardHandler(this, null) {
@Override
protected NamedList<Object> request(String url, SolrRequest req) throws IOException, SolrServerException {
try (SolrClient client = new HttpSolrClient.Builder(url).withHttpClient(httpClient).build()) {
return client.request(req);
}
}
};
}
@Override
public void init(PluginInfo info) {
StringBuilder sb = new StringBuilder();
NamedList args = info.initArgs;
this.soTimeout = getParameter(args, HttpClientUtil.PROP_SO_TIMEOUT, soTimeout,sb);
this.scheme = getParameter(args, INIT_URL_SCHEME, null,sb);
if(StringUtils.endsWith(this.scheme, "://")) {
this.scheme = StringUtils.removeEnd(this.scheme, "://");
@ -171,9 +177,6 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
"Unknown metricNameStrategy: " + strategy + " found. Must be one of: " + KNOWN_METRIC_NAME_STRATEGIES.keySet());
}
this.connectionTimeout = getParameter(args, HttpClientUtil.PROP_CONNECTION_TIMEOUT, connectionTimeout, sb);
this.maxConnectionsPerHost = getParameter(args, HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost,sb);
this.maxConnections = getParameter(args, HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections,sb);
this.corePoolSize = getParameter(args, INIT_CORE_POOL_SIZE, corePoolSize,sb);
this.maximumPoolSize = getParameter(args, INIT_MAX_POOL_SIZE, maximumPoolSize,sb);
this.keepAliveTime = getParameter(args, MAX_THREAD_IDLE_TIME, keepAliveTime,sb);
@ -209,31 +212,25 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
new DefaultSolrThreadFactory("httpShardExecutor")
);
ModifiableSolrParams clientParams = getClientParams();
httpRequestExecutor = new InstrumentedHttpRequestExecutor(this.metricNameStrategy);
clientConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry());
this.defaultClient = HttpClientUtil.createClient(clientParams, clientConnectionManager, false, httpRequestExecutor);
this.loadbalancer = createLoadbalancer(defaultClient);
this.httpListenerFactory = new InstrumentedHttpListenerFactory(this.metricNameStrategy);
int connectionTimeout = getParameter(args, HttpClientUtil.PROP_CONNECTION_TIMEOUT,
HttpClientUtil.DEFAULT_CONNECT_TIMEOUT, sb);
int maxConnectionsPerHost = getParameter(args, HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST,
HttpClientUtil.DEFAULT_MAXCONNECTIONSPERHOST, sb);
int soTimeout = getParameter(args, HttpClientUtil.PROP_SO_TIMEOUT,
HttpClientUtil.DEFAULT_SO_TIMEOUT, sb);
this.defaultClient = new Http2SolrClient.Builder()
.connectionTimeout(connectionTimeout)
.idleTimeout(soTimeout)
.maxConnectionsPerHost(maxConnectionsPerHost).build();
this.defaultClient.addListenerFactory(this.httpListenerFactory);
this.loadbalancer = new LBHttp2SolrClient(defaultClient);
}
protected ModifiableSolrParams getClientParams() {
ModifiableSolrParams clientParams = new ModifiableSolrParams();
clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, maxConnectionsPerHost);
clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, maxConnections);
return clientParams;
}
protected ExecutorService getThreadPoolExecutor(){
return this.commExecutor;
}
protected LBHttpSolrClient createLoadbalancer(HttpClient httpClient){
LBHttpSolrClient client = new Builder()
.withHttpClient(httpClient)
.withConnectionTimeout(connectionTimeout)
.withSocketTimeout(soTimeout)
.build();
return client;
@Override
public void setSecurityBuilder(HttpClientBuilderPlugin clientBuilderPlugin) {
clientBuilderPlugin.setup(defaultClient);
}
protected <T> T getParameter(NamedList initArgs, String configKey, T defaultValue, StringBuilder sb) {
@ -258,10 +255,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
}
} finally {
if (defaultClient != null) {
HttpClientUtil.close(defaultClient);
}
if (clientConnectionManager != null) {
clientConnectionManager.close();
IOUtils.closeQuietly(defaultClient);
}
}
}
@ -274,17 +268,17 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
* @param urls The list of solr server urls to load balance across
* @return The response from the request
*/
public LBHttpSolrClient.Rsp makeLoadBalancedRequest(final QueryRequest req, List<String> urls)
public LBSolrClient.Rsp makeLoadBalancedRequest(final QueryRequest req, List<String> urls)
throws SolrServerException, IOException {
return loadbalancer.request(newLBHttpSolrClientReq(req, urls));
}
protected LBHttpSolrClient.Req newLBHttpSolrClientReq(final QueryRequest req, List<String> urls) {
protected LBSolrClient.Req newLBHttpSolrClientReq(final QueryRequest req, List<String> urls) {
int numServersToTry = (int)Math.floor(urls.size() * this.permittedLoadBalancerRequestsMaximumFraction);
if (numServersToTry < this.permittedLoadBalancerRequestsMinimumAbsolute) {
numServersToTry = this.permittedLoadBalancerRequestsMinimumAbsolute;
}
return new LBHttpSolrClient.Req(req, urls, numServersToTry);
return new LBSolrClient.Req(req, urls, numServersToTry);
}
/**
@ -305,7 +299,7 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
}
/**
* A distributed request is made via {@link LBHttpSolrClient} to the first live server in the URL list.
* A distributed request is made via {@link LBSolrClient} to the first live server in the URL list.
* This means it is just as likely to choose current host as any of the other hosts.
* This function makes sure that the cores are sorted according to the given list of preferences.
* E.g. If all nodes prefer local cores then a bad/heavily-loaded node will receive less requests from
@ -472,10 +466,8 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory implements org.
@Override
public void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) {
this.metricTag = tag;
String expandedScope = SolrMetricManager.mkName(scope, SolrInfoBean.Category.QUERY.name());
clientConnectionManager.initializeMetrics(manager, registry, tag, expandedScope);
httpRequestExecutor.initializeMetrics(manager, registry, tag, expandedScope);
httpListenerFactory.initializeMetrics(manager, registry, tag, expandedScope);
commExecutor = MetricUtils.instrumentedExecutorService(commExecutor, null,
manager.registry(registry),
SolrMetricManager.mkName("httpShardExecutor", expandedScope, "threadPool"));

10
solr/core/src/java/org/apache/solr/handler/component/ShardHandlerFactory.java
View File

@ -15,21 +15,25 @@
* limitations under the License.
*/
package org.apache.solr.handler.component;
import java.util.Collections;
import java.util.Locale;
import com.google.common.collect.ImmutableMap;
import org.apache.solr.common.SolrException;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.security.HttpClientBuilderPlugin;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import java.util.Collections;
import java.util.Locale;
public abstract class ShardHandlerFactory {
public abstract ShardHandler getShardHandler();
public abstract void close();
public void setSecurityBuilder(HttpClientBuilderPlugin clientBuilderPlugin){};
/**
* Create a new ShardHandlerFactory instance
* @param info a PluginInfo object defining which type to create. If null,

19
solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java
View File

@ -35,6 +35,7 @@ import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.http.HttpRequest;
import org.apache.http.protocol.HttpContext;
import org.eclipse.jetty.client.api.Request;
/**
*
@ -117,6 +118,24 @@ public abstract class AuthenticationPlugin implements Closeable, SolrInfoBean, S
protected boolean interceptInternodeRequest(HttpRequest httpRequest, HttpContext httpContext) {
return this instanceof HttpClientBuilderPlugin;
}
/**
* Override this method to intercept internode requests. This allows your authentication
* plugin to decide on per-request basis whether it should handle inter-node requests or
* delegate to {@link PKIAuthenticationPlugin}. Return true to indicate that your plugin
* did handle the request, or false to signal that PKI plugin should handle it. This method
* will be called by {@link PKIAuthenticationPlugin}'s interceptor.
*
* <p>
* If not overridden, this method will return true for plugins implementing {@link HttpClientBuilderPlugin}.
* This method can be overridden by subclasses e.g. to set HTTP headers, even if you don't use a clientBuilder.
* </p>
* @param request the httpRequest that is about to be sent to another internal Solr node
* @return true if this plugin handled authentication for the request, else false
*/
protected boolean interceptInternodeRequest(Request request) {
return this instanceof HttpClientBuilderPlugin;
}
/**
* Cleanup any per request data

16
solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java
View File

@ -46,11 +46,13 @@ import org.apache.http.annotation.ThreadingBehavior;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.message.BasicHeader;
import org.apache.http.protocol.HttpContext;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SpecProvider;
import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.ValidatingJsonMap;
import org.eclipse.jetty.client.api.Request;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -233,6 +235,20 @@ public class BasicAuthPlugin extends AuthenticationPlugin implements ConfigEdita
return false;
}
@Override
protected boolean interceptInternodeRequest(Request request) {
if (forwardCredentials) {
Object userToken = request.getAttributes().get(Http2SolrClient.REQ_PRINCIPAL_KEY);
if (userToken instanceof BasicAuthUserPrincipal) {
BasicAuthUserPrincipal principal = (BasicAuthUserPrincipal) userToken;
String userPassBase64 = Base64.encodeBase64String((principal.getName() + ":" + principal.getPassword()).getBytes(StandardCharsets.UTF_8));
request.header(HttpHeaders.AUTHORIZATION, "Basic " + userPassBase64);
return true;
}
}
return false;
}
@Override
public ValidatingJsonMap getSpec() {
return authenticationProvider.getSpec();

6
solr/core/src/java/org/apache/solr/security/ConfigurableInternodeAuthHadoopPlugin.java
View File

@ -21,6 +21,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.HttpClientBuilderFactory;
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.core.CoreContainer;
@ -52,6 +53,11 @@ public class ConfigurableInternodeAuthHadoopPlugin extends HadoopAuthPlugin impl
factory = this.coreContainer.getResourceLoader().newInstance(httpClientBuilderFactory, HttpClientBuilderFactory.class);
}
@Override
public void setup(Http2SolrClient client) {
factory.setup(client);
}
@Override
public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) {
return factory.getHttpClientBuilder(Optional.ofNullable(builder));

5
solr/core/src/java/org/apache/solr/security/DelegationTokenKerberosFilter.java
View File

@ -21,6 +21,7 @@ import java.lang.invoke.MethodHandles;
import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
@ -61,6 +62,7 @@ public class DelegationTokenKerberosFilter extends DelegationTokenAuthentication
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private CuratorFramework curatorFramework;
private final Locale defaultLocale = Locale.getDefault();
@Override
public void init(FilterConfig conf) throws ServletException {
@ -118,6 +120,7 @@ public class DelegationTokenKerberosFilter extends DelegationTokenAuthentication
@Override
public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse)
throws IOException, ServletException {
Locale.setDefault(defaultLocale);
HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
UserGroupInformation ugi = HttpUserGroupInformation.get();
@ -131,6 +134,8 @@ public class DelegationTokenKerberosFilter extends DelegationTokenAuthentication
}
};
// A hack until HADOOP-15681 get committed
Locale.setDefault(Locale.US);
super.doFilter(requestNonNullQueryString, response, filterChainWrapper);
}

5
solr/core/src/java/org/apache/solr/security/HadoopAuthFilter.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.security;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
@ -58,6 +59,7 @@ public class HadoopAuthFilter extends DelegationTokenAuthenticationFilter {
static final String DELEGATION_TOKEN_ZK_CLIENT = "solr.kerberos.delegation.token.zk.client";
private CuratorFramework curatorFramework;
private final Locale defaultLocale = Locale.getDefault();
@Override
public void init(FilterConfig conf) throws ServletException {
@ -94,6 +96,7 @@ public class HadoopAuthFilter extends DelegationTokenAuthenticationFilter {
@Override
public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse)
throws IOException, ServletException {
Locale.setDefault(defaultLocale);
HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
UserGroupInformation ugi = HttpUserGroupInformation.get();
@ -107,6 +110,8 @@ public class HadoopAuthFilter extends DelegationTokenAuthenticationFilter {
}
};
// A hack until HADOOP-15681 get committed
Locale.setDefault(Locale.US);
super.doFilter(requestNonNullQueryString, response, filterChainWrapper);
}

17
solr/core/src/java/org/apache/solr/security/HadoopAuthPlugin.java
View File

@ -26,6 +26,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import javax.servlet.FilterChain;
@ -120,6 +121,7 @@ public class HadoopAuthPlugin extends AuthenticationPlugin {
private static final boolean TRACE_HTTP = Boolean.getBoolean("hadoopauth.tracehttp");
private AuthenticationFilter authFilter;
private final Locale defaultLocale = Locale.getDefault();
protected final CoreContainer coreContainer;
public HadoopAuthPlugin(CoreContainer coreContainer) {
@ -130,7 +132,20 @@ public class HadoopAuthPlugin extends AuthenticationPlugin {
public void init(Map<String,Object> pluginConfig) {
try {
String delegationTokenEnabled = (String)pluginConfig.getOrDefault(DELEGATION_TOKEN_ENABLED_PROPERTY, "false");
authFilter = (Boolean.parseBoolean(delegationTokenEnabled)) ? new HadoopAuthFilter() : new AuthenticationFilter();
authFilter = (Boolean.parseBoolean(delegationTokenEnabled)) ? new HadoopAuthFilter() : new AuthenticationFilter() {
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) throws IOException, ServletException {
// A hack until HADOOP-15681 get committed
Locale.setDefault(Locale.US);
super.doFilter(request, response, filterChain);
}
@Override
protected void doFilter(FilterChain filterChain, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
Locale.setDefault(defaultLocale);
super.doFilter(filterChain, request, response);
}
};
// Initialize kerberos before initializing curator instance.
boolean initKerberosZk = Boolean.parseBoolean((String)pluginConfig.getOrDefault(INIT_KERBEROS_ZK, "false"));

5
solr/core/src/java/org/apache/solr/security/HttpClientBuilderPlugin.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.solr.security;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
/**
@ -34,4 +35,8 @@ public interface HttpClientBuilderPlugin {
* @lucene.experimental
*/
public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder);
public default void setup(Http2SolrClient client) {
}
}

8
solr/core/src/java/org/apache/solr/security/KerberosFilter.java
View File

@ -17,6 +17,7 @@
package org.apache.solr.security;
import java.io.IOException;
import java.util.Locale;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
@ -29,7 +30,9 @@ import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authentication.server.AuthenticationHandler;
public class KerberosFilter extends AuthenticationFilter {
private final Locale defaultLocale = Locale.getDefault();
@Override
public void init(FilterConfig conf) throws ServletException {
super.init(conf);
@ -51,12 +54,15 @@ public class KerberosFilter extends AuthenticationFilter {
@Override
protected void doFilter(FilterChain filterChain, HttpServletRequest request,
HttpServletResponse response) throws IOException, ServletException {
Locale.setDefault(defaultLocale);
super.doFilter(filterChain, request, response);
}
@Override
public void doFilter(ServletRequest request, ServletResponse response,
FilterChain filterChain) throws IOException, ServletException {
// A hack until HADOOP-15681 get committed
Locale.setDefault(Locale.US);
super.doFilter(request, response, filterChain);
}
}

6
solr/core/src/java/org/apache/solr/security/KerberosPlugin.java
View File

@ -35,6 +35,7 @@ import javax.servlet.http.HttpServletResponseWrapper;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.collections.iterators.IteratorEnumeration;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.cloud.ZkController;
@ -255,6 +256,11 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu
return kerberosBuilder.getBuilder(builder);
}
@Override
public void setup(Http2SolrClient client) {
kerberosBuilder.setup(client);
}
@Override
public void close() {
kerberosFilter.destroy();

37
solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java
View File

@ -28,6 +28,7 @@ import java.security.Principal;
import java.security.PublicKey;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.http.HttpEntity;
@ -39,7 +40,9 @@ import org.apache.http.auth.BasicUserPrincipal;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpListenerFactory;
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.common.util.Base64;
import org.apache.solr.common.util.ExecutorUtil;
@ -49,12 +52,12 @@ import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.util.CryptoKeys;
import org.eclipse.jetty.client.api.Request;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.nio.charset.StandardCharsets.UTF_8;
public class PKIAuthenticationPlugin extends AuthenticationPlugin implements HttpClientBuilderPlugin {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final Map<String, PublicKey> keyCache = new ConcurrentHashMap<>();
@ -223,6 +226,25 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt
}
@Override
public void setup(Http2SolrClient client) {
final HttpListenerFactory.RequestResponseListener listener = new HttpListenerFactory.RequestResponseListener() {
@Override
public void onQueued(Request request) {
if (cores.getAuthenticationPlugin() == null) {
return;
}
if (!cores.getAuthenticationPlugin().interceptInternodeRequest(request)) {
log.debug("{} secures this internode request", this.getClass().getSimpleName());
generateToken().ifPresent(s -> request.header(HEADER, myNodeName + " " + s));
} else {
log.debug("{} secures this internode request", cores.getAuthenticationPlugin().getClass().getSimpleName());
}
}
};
client.addListenerFactory(() -> listener);
}
@Override
public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) {
HttpClientUtil.addRequestInterceptor(interceptor);
@ -254,15 +276,16 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt
}
@SuppressForbidden(reason = "Needs currentTimeMillis to set current time in header")
void setHeader(HttpRequest httpRequest) {
private Optional<String> generateToken() {
SolrRequestInfo reqInfo = getRequestInfo();
String usr;
if (reqInfo != null) {
Principal principal = reqInfo.getReq().getUserPrincipal();
if (principal == null) {
log.debug("principal is null");
//this had a request but not authenticated
//so we don't not need to set a principal
return;
return Optional.empty();
} else {
usr = principal.getName();
}
@ -270,7 +293,7 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt
if (!isSolrThread()) {
//if this is not running inside a Solr threadpool (as in testcases)
// then no need to add any header
return;
return Optional.empty();
}
//this request seems to be originated from Solr itself
usr = "$"; //special name to denote the user is the node itself
@ -281,7 +304,11 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt
byte[] payload = s.getBytes(UTF_8);
byte[] payloadCipher = publicKeyHandler.keyPair.encrypt(ByteBuffer.wrap(payload));
String base64Cipher = Base64.byteArrayToBase64(payloadCipher);
httpRequest.setHeader(HEADER, myNodeName + " " + base64Cipher);
return Optional.of(base64Cipher);
}
void setHeader(HttpRequest httpRequest) {
generateToken().ifPresent(s -> httpRequest.setHeader(HEADER, myNodeName + " " + s));
}
boolean isSolrThread() {

1
solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
View File

@ -574,6 +574,7 @@ public class HttpSolrCall {
}
}
//TODO using Http2Client
private void remoteQuery(String coreUrl, HttpServletResponse resp) throws IOException {
HttpRequestBase method = null;
HttpEntity httpEntity = null;

1
solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
View File

@ -93,6 +93,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
protected final CountDownLatch init = new CountDownLatch(1);
protected String abortErrorMessage = null;
//TODO using Http2Client
protected HttpClient httpClient;
private ArrayList<Pattern> excludePatterns;

41
solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
View File

@ -33,13 +33,11 @@ import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
import org.apache.http.HttpResponse;
import org.apache.http.NoHttpResponseException;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrException;
@ -244,15 +242,13 @@ public class SolrCmdDistributor implements Closeable {
// we need to do any retries before commit...
blockAndDoRetries();
UpdateRequest uReq = new UpdateRequest();
uReq.setParams(params);
addCommit(uReq, cmd);
log.debug("Distrib commit to: {} params: {}", nodes, params);
for (Node node : nodes) {
UpdateRequest uReq = new UpdateRequest();
uReq.setParams(params);
addCommit(uReq, cmd);
submit(new Req(cmd, node, uReq, false), true);
}
@ -292,8 +288,9 @@ public class SolrCmdDistributor implements Closeable {
if (req.synchronous) {
blockAndDoRetries();
try (HttpSolrClient client = new HttpSolrClient.Builder(req.node.getUrl()).withHttpClient(clients.getHttpClient()).build()) {
client.request(req.uReq);
try {
req.uReq.setBasePath(req.node.getUrl());
clients.getHttpClient().request(req.uReq);
} catch (Exception e) {
SolrException.log(log, e);
Error error = new Error();
@ -394,11 +391,11 @@ public class SolrCmdDistributor implements Closeable {
//
// In the case of a leaderTracker and rollupTracker both being present, then we need to take care when assembling
// the final response to check both the rollup and leader trackers on the aggregator node.
public void trackRequestResult(HttpResponse resp, boolean success) {
public void trackRequestResult(org.eclipse.jetty.client.api.Response resp, InputStream respBody, boolean success) {
// Returning Integer.MAX_VALUE here means there was no "rf" on the response, therefore we just need to increment
// our achieved rf if we are a leader, i.e. have a leaderTracker.
int rfFromResp = getRfFromResponse(resp);
int rfFromResp = getRfFromResponse(respBody);
if (leaderTracker != null && rfFromResp == Integer.MAX_VALUE) {
leaderTracker.trackRequestResult(node, success);
@ -409,13 +406,9 @@ public class SolrCmdDistributor implements Closeable {
}
}
private int getRfFromResponse(HttpResponse resp) {
if (resp != null) {
InputStream inputStream = null;
private int getRfFromResponse(InputStream inputStream) {
if (inputStream != null) {
try {
inputStream = resp.getEntity().getContent();
BinaryResponseParser brp = new BinaryResponseParser();
NamedList<Object> nl = brp.processResponse(inputStream, null);
Object hdr = nl.get("responseHeader");
@ -429,11 +422,9 @@ public class SolrCmdDistributor implements Closeable {
} catch (Exception e) {
log.warn("Failed to parse response from {} during replication factor accounting", node, e);
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (Exception ignore) {
}
try {
inputStream.close();
} catch (Exception ignore) {
}
}
}

78
solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
View File

@ -16,26 +16,21 @@
*/
package org.apache.solr.update;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.common.SolrException;
import org.apache.solr.update.SolrCmdDistributor.Error;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.eclipse.jetty.client.api.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -43,56 +38,41 @@ public class StreamingSolrClients {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final int runnerCount = Integer.getInteger("solr.cloud.replication.runners", 1);
private HttpClient httpClient;
private Map<String, ConcurrentUpdateSolrClient> solrClients = new HashMap<>();
private Http2SolrClient httpClient;
private Map<String, ConcurrentUpdateHttp2SolrClient> solrClients = new HashMap<>();
private List<Error> errors = Collections.synchronizedList(new ArrayList<Error>());
private ExecutorService updateExecutor;
private int socketTimeout;
private int connectionTimeout;
public StreamingSolrClients(UpdateShardHandler updateShardHandler) {
this.updateExecutor = updateShardHandler.getUpdateExecutor();
httpClient = updateShardHandler.getUpdateOnlyHttpClient();
socketTimeout = updateShardHandler.getSocketTimeout();
connectionTimeout = updateShardHandler.getConnectionTimeout();
this.httpClient = updateShardHandler.getUpdateOnlyHttpClient();
}
public List<Error> getErrors() {
return errors;
}
public void clearErrors() {
errors.clear();
}
public synchronized SolrClient getSolrClient(final SolrCmdDistributor.Req req) {
String url = getFullUrl(req.node.getUrl());
ConcurrentUpdateSolrClient client = solrClients.get(url);
ConcurrentUpdateHttp2SolrClient client = solrClients.get(url);
if (client == null) {
// NOTE: increasing to more than 1 threadCount for the client could cause updates to be reordered
// on a greater scale since the current behavior is to only increase the number of connections/Runners when
// the queue is more than half full.
client = new ErrorReportingConcurrentUpdateSolrClient.Builder(url, req, errors)
.withHttpClient(httpClient)
client = new ErrorReportingConcurrentUpdateSolrClient.Builder(url, httpClient, req, errors)
.withQueueSize(100)
.withThreadCount(runnerCount)
.withExecutorService(updateExecutor)
.alwaysStreamDeletes()
.withSocketTimeout(socketTimeout)
.withConnectionTimeout(connectionTimeout)
.build();
client.setPollQueueTime(Integer.MAX_VALUE); // minimize connections created
client.setParser(new BinaryResponseParser());
client.setRequestWriter(new BinaryRequestWriter());
Set<String> queryParams = new HashSet<>(2);
queryParams.add(DistributedUpdateProcessor.DISTRIB_FROM);
queryParams.add(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM);
client.setQueryParams(queryParams);
solrClients.put(url, client);
}
@ -100,17 +80,17 @@ public class StreamingSolrClients {
}
public synchronized void blockUntilFinished() {
for (ConcurrentUpdateSolrClient client : solrClients.values()) {
for (ConcurrentUpdateHttp2SolrClient client : solrClients.values()) {
client.blockUntilFinished();
}
}
public synchronized void shutdown() {
for (ConcurrentUpdateSolrClient client : solrClients.values()) {
for (ConcurrentUpdateHttp2SolrClient client : solrClients.values()) {
client.close();
}
}
private String getFullUrl(String url) {
String fullUrl;
if (!url.startsWith("http://") && !url.startsWith("https://")) {
@ -121,26 +101,26 @@ public class StreamingSolrClients {
return fullUrl;
}
public HttpClient getHttpClient() {
public Http2SolrClient getHttpClient() {
return httpClient;
}
public ExecutorService getUpdateExecutor() {
return updateExecutor;
}
}
class ErrorReportingConcurrentUpdateSolrClient extends ConcurrentUpdateSolrClient {
class ErrorReportingConcurrentUpdateSolrClient extends ConcurrentUpdateHttp2SolrClient {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final SolrCmdDistributor.Req req;
private final List<Error> errors;
public ErrorReportingConcurrentUpdateSolrClient(Builder builder) {
super(builder);
this.req = builder.req;
this.errors = builder.errors;
}
@Override
public void handleError(Throwable ex) {
log.error("error", ex);
@ -153,20 +133,20 @@ class ErrorReportingConcurrentUpdateSolrClient extends ConcurrentUpdateSolrClien
errors.add(error);
if (!req.shouldRetry(error)) {
// only track the error if we are not retrying the request
req.trackRequestResult(null, false);
req.trackRequestResult(null, null, false);
}
}
@Override
public void onSuccess(HttpResponse resp) {
req.trackRequestResult(resp, true);
public void onSuccess(Response resp, InputStream respBody) {
req.trackRequestResult(resp, respBody, true);
}
static class Builder extends ConcurrentUpdateSolrClient.Builder {
static class Builder extends ConcurrentUpdateHttp2SolrClient.Builder {
protected SolrCmdDistributor.Req req;
protected List<Error> errors;
public Builder(String baseSolrUrl, SolrCmdDistributor.Req req, List<Error> errors) {
super(baseSolrUrl);
public Builder(String baseSolrUrl, Http2SolrClient client, SolrCmdDistributor.Req req, List<Error> errors) {
super(baseSolrUrl, client);
this.req = req;
this.errors = errors;
}

92
solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
View File

@ -17,6 +17,7 @@
package org.apache.solr.update;
import java.lang.invoke.MethodHandles;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
@ -25,18 +26,25 @@ import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.security.HttpClientBuilderPlugin;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.apache.solr.util.stats.HttpClientMetricNameStrategy;
import org.apache.solr.util.stats.InstrumentedHttpListenerFactory;
import org.apache.solr.util.stats.InstrumentedHttpRequestExecutor;
import org.apache.solr.util.stats.InstrumentedPoolingHttpClientConnectionManager;
import org.apache.solr.util.stats.MetricUtils;
@ -64,7 +72,7 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
private ExecutorService recoveryExecutor;
private final CloseableHttpClient updateOnlyClient;
private final Http2SolrClient updateOnlyClient;
private final CloseableHttpClient recoveryOnlyClient;
@ -78,6 +86,8 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
private final InstrumentedHttpRequestExecutor httpRequestExecutor;
private final InstrumentedHttpListenerFactory updateHttpListenerFactory;
private final Set<String> metricNames = ConcurrentHashMap.newKeySet();
private MetricRegistry registry;
@ -89,6 +99,7 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
updateOnlyConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry());
recoveryOnlyConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry());
defaultConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSchemaRegisteryProvider().getSchemaRegistry());
ModifiableSolrParams clientParams = new ModifiableSolrParams();
if (cfg != null ) {
updateOnlyConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
updateOnlyConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
@ -96,38 +107,35 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
recoveryOnlyConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
defaultConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
defaultConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
}
ModifiableSolrParams clientParams = new ModifiableSolrParams();
if (cfg != null) {
clientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, cfg.getDistributedSocketTimeout());
clientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, cfg.getDistributedConnectionTimeout());
// following is done only for logging complete configuration.
// The maxConnections and maxConnectionsPerHost have already been specified on the connection manager
clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, cfg.getMaxUpdateConnections());
clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, cfg.getMaxUpdateConnectionsPerHost());
socketTimeout = cfg.getDistributedSocketTimeout();
connectionTimeout = cfg.getDistributedConnectionTimeout();
}
HttpClientMetricNameStrategy metricNameStrategy = KNOWN_METRIC_NAME_STRATEGIES.get(UpdateShardHandlerConfig.DEFAULT_METRICNAMESTRATEGY);
if (cfg != null) {
metricNameStrategy = KNOWN_METRIC_NAME_STRATEGIES.get(cfg.getMetricNameStrategy());
if (metricNameStrategy == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unknown metricNameStrategy: " + cfg.getMetricNameStrategy() + " found. Must be one of: " + KNOWN_METRIC_NAME_STRATEGIES.keySet());
}
}
log.debug("Created default UpdateShardHandler HTTP client with params: {}", clientParams);
httpRequestExecutor = new InstrumentedHttpRequestExecutor(metricNameStrategy);
updateOnlyClient = HttpClientUtil.createClient(clientParams, updateOnlyConnectionManager, false, httpRequestExecutor);
httpRequestExecutor = new InstrumentedHttpRequestExecutor(getMetricNameStrategy(cfg));
updateHttpListenerFactory = new InstrumentedHttpListenerFactory(getNameStrategy(cfg));
recoveryOnlyClient = HttpClientUtil.createClient(clientParams, recoveryOnlyConnectionManager, false, httpRequestExecutor);
defaultClient = HttpClientUtil.createClient(clientParams, defaultConnectionManager, false, httpRequestExecutor);
// following is done only for logging complete configuration.
// The maxConnections and maxConnectionsPerHost have already been specified on the connection manager
if (cfg != null) {
clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS, cfg.getMaxUpdateConnections());
clientParams.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, cfg.getMaxUpdateConnectionsPerHost());
Http2SolrClient.Builder updateOnlyClientBuilder = new Http2SolrClient.Builder();
if (cfg != null) {
updateOnlyClientBuilder
.connectionTimeout(cfg.getDistributedConnectionTimeout())
.idleTimeout(cfg.getDistributedSocketTimeout())
.maxConnectionsPerHost(cfg.getMaxUpdateConnectionsPerHost());
}
log.debug("Created default UpdateShardHandler HTTP client with params: {}", clientParams);
log.debug("Created update only UpdateShardHandler HTTP client with params: {}", clientParams);
updateOnlyClient = updateOnlyClientBuilder.build();
updateOnlyClient.addListenerFactory(updateHttpListenerFactory);
Set<String> queryParams = new HashSet<>(2);
queryParams.add(DistributedUpdateProcessor.DISTRIB_FROM);
queryParams.add(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM);
updateOnlyClient.setQueryParams(queryParams);
ThreadFactory recoveryThreadFactory = new SolrjNamedThreadFactory("recoveryExecutor");
if (cfg != null && cfg.getMaxRecoveryThreads() > 0) {
@ -139,6 +147,32 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
}
}
private HttpClientMetricNameStrategy getMetricNameStrategy(UpdateShardHandlerConfig cfg) {
HttpClientMetricNameStrategy metricNameStrategy = KNOWN_METRIC_NAME_STRATEGIES.get(UpdateShardHandlerConfig.DEFAULT_METRICNAMESTRATEGY);
if (cfg != null) {
metricNameStrategy = KNOWN_METRIC_NAME_STRATEGIES.get(cfg.getMetricNameStrategy());
if (metricNameStrategy == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unknown metricNameStrategy: " + cfg.getMetricNameStrategy() + " found. Must be one of: " + KNOWN_METRIC_NAME_STRATEGIES.keySet());
}
}
return metricNameStrategy;
}
private InstrumentedHttpListenerFactory.NameStrategy getNameStrategy(UpdateShardHandlerConfig cfg) {
InstrumentedHttpListenerFactory.NameStrategy nameStrategy =
InstrumentedHttpListenerFactory.KNOWN_METRIC_NAME_STRATEGIES.get(UpdateShardHandlerConfig.DEFAULT_METRICNAMESTRATEGY);
if (cfg != null) {
nameStrategy = InstrumentedHttpListenerFactory.KNOWN_METRIC_NAME_STRATEGIES.get(cfg.getMetricNameStrategy());
if (nameStrategy == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unknown metricNameStrategy: " + cfg.getMetricNameStrategy() + " found. Must be one of: " + KNOWN_METRIC_NAME_STRATEGIES.keySet());
}
}
return nameStrategy;
}
@Override
public String getName() {
return this.getClass().getName();
@ -148,7 +182,7 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) {
registry = manager.registry(registryName);
String expandedScope = SolrMetricManager.mkName(scope, getCategory().name());
updateOnlyConnectionManager.initializeMetrics(manager, registryName, tag, expandedScope);
updateHttpListenerFactory.initializeMetrics(manager, registryName, tag, expandedScope);
defaultConnectionManager.initializeMetrics(manager, registryName, tag, expandedScope);
updateExecutor = MetricUtils.instrumentedExecutorService(updateExecutor, this, registry,
SolrMetricManager.mkName("updateOnlyExecutor", expandedScope, "threadPool"));
@ -182,7 +216,7 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
}
// don't introduce a bug, this client is for sending updates only!
public HttpClient getUpdateOnlyHttpClient() {
public Http2SolrClient getUpdateOnlyHttpClient() {
return updateOnlyClient;
}
@ -225,21 +259,25 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
HttpClientUtil.close(updateOnlyClient);
IOUtils.closeQuietly(updateOnlyClient);
HttpClientUtil.close(recoveryOnlyClient);
HttpClientUtil.close(defaultClient);
updateOnlyConnectionManager.close();
defaultConnectionManager.close();
recoveryOnlyConnectionManager.close();
}
}
@VisibleForTesting
public int getSocketTimeout() {
return socketTimeout;
}
@VisibleForTesting
public int getConnectionTimeout() {
return connectionTimeout;
}
public void setSecurityBuilder(HttpClientBuilderPlugin builder) {
builder.setup(updateOnlyClient);
}
}

38
solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
View File

@ -43,7 +43,6 @@ import org.apache.lucene.util.CharsRefBuilder;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrRequest.METHOD;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.SimpleSolrResponse;
@ -119,7 +118,6 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
* Request forwarded to a leader of a different shard will be retried up to this amount of times by default
*/
static final int MAX_RETRIES_ON_FORWARD_DEAULT = Integer.getInteger("solr.retries.on.forward", 25);
/**
* Requests from leader to it's followers will be retried this amount of times by default
*/
@ -153,7 +151,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// used to assert we don't call finish more than once, see finish()
private boolean finished = false;
private final SolrQueryRequest req;
private final SolrQueryResponse rsp;
private final UpdateRequestProcessor next;
@ -168,9 +166,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
private NamedList<Object> deleteResponse = null;
private NamedList<Object> deleteByQueryResponse = null;
private CharsRefBuilder scratch;
private final SchemaField idField;
private SolrCmdDistributor cmdDistrib;
private final boolean zkEnabled;
@ -178,7 +176,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
private final CloudDescriptor cloudDesc;
private final String collection;
private final ZkController zkController;
// these are setup at the start of each request processing
// method in this update processor
private boolean isLeader = true;
@ -187,7 +185,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
private List<Node> nodes;
private Set<String> skippedCoreNodeNames;
private boolean isIndexChanged = false;
/**
* Number of times requests forwarded to some other shard's leader can be retried
*/
@ -198,7 +196,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
private final int maxRetriesToFollowers = MAX_RETRIES_TO_FOLLOWERS_DEFAULT;
private UpdateCommand updateCommand; // the current command this processor is working on.
//used for keeping track of replicas that have processed an add/update from the leader
private RollupRequestReplicationTracker rollupReplicationTracker = null;
private LeaderRequestReplicationTracker leaderReplicationTracker = null;
@ -234,7 +232,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// SolrRequestInfo reqInfo = returnVersions ? SolrRequestInfo.getRequestInfo() : null;
this.req = req;
// this should always be used - see filterParams
DistributedUpdateProcessorFactory.addParamToDistributedRequestWhitelist
(this.req, UpdateParams.UPDATE_CHAIN, TEST_DISTRIB_SKIP_SERVERS, CommonParams.VERSION_FIELD);
@ -248,7 +246,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
}
//this.rsp = reqInfo != null ? reqInfo.getRsp() : null;
cloudDesc = req.getCore().getCoreDescriptor().getCloudDescriptor();
if (cloudDesc != null) {
collection = cloudDesc.getCollectionName();
replicaType = cloudDesc.getReplicaType();
@ -620,6 +618,18 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
}
}
int count = 0;
while (((isLeader && !localIsLeader) || (isSubShardLeader && !localIsLeader)) && count < 5) {
count++;
// re-getting localIsLeader since we published to ZK first before setting localIsLeader value
localIsLeader = cloudDesc.isLeader();
try {
Thread.sleep(500);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
if ((isLeader && !localIsLeader) || (isSubShardLeader && !localIsLeader)) {
log.error("ClusterState says we are the leader, but locally we don't think so");
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE,
@ -1343,10 +1353,10 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
leaderUrl = leader.getCoreUrl();
}
NamedList<Object> rsp = null;
try (HttpSolrClient hsc = new HttpSolrClient.Builder(leaderUrl).
withHttpClient(updateShardHandler.getUpdateOnlyHttpClient()).build()) {
rsp = hsc.request(ur);
NamedList<Object> rsp;
try {
ur.setBasePath(leaderUrl);
rsp = updateShardHandler.getUpdateOnlyHttpClient().request(ur);
} catch (SolrServerException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Error during fetching [" + id +
"] from leader (" + leaderUrl + "): ", e);

26
solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java
View File

@ -166,12 +166,12 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
public void processDelete(DeleteUpdateCommand cmd) throws IOException {
try {
super.processDelete(cmd);
} catch (Throwable t) {
firstErrTracker.caught(t);
ToleratedUpdateError err = new ToleratedUpdateError(cmd.isDeleteById() ? CmdType.DELID : CmdType.DELQ,
cmd.isDeleteById() ? cmd.id : cmd.query,
t.getMessage());
@ -184,7 +184,7 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
if (CmdType.DELQ.equals(err.getType())) {
knownDBQErrors.add(err);
}
if (knownErrors.size() > maxErrors) {
firstErrTracker.throwFirst();
}
@ -233,13 +233,13 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
// even if processAdd threw an error, this.finish() is still called and we might have additional
// errors from other remote leaders that we need to check for from the finish method of downstream processors
// (like DUP)
try {
super.finish();
} catch (DistributedUpdateProcessor.DistributedUpdatesAsyncException duae) {
firstErrTracker.caught(duae);
// adjust our stats based on each of the distributed errors
for (Error error : duae.errors) {
// we can't trust the req info from the Error, because multiple original requests might have been
@ -276,7 +276,7 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
knownDBQErrors.add(err);
}
}
knownErrors.add(err);
}
}
@ -290,7 +290,7 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
firstErrTracker.annotate(knownErrors);
// decide if we have hit a situation where we know an error needs to be thrown.
if ((DistribPhase.TOLEADER.equals(distribPhase) ? 0 : maxErrors) < knownErrors.size()) {
// NOTE: even if maxErrors wasn't exceeded, we need to throw an error when we have any errors if we're
// a leader that was forwarded to by another node so that the forwarding node knows we encountered some
@ -327,11 +327,11 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
SolrException first = null;
boolean thrown = false;
public FirstErrTracker() {
/* NOOP */
}
/**
* Call this method immediately anytime an exception is caught from a down stream method --
* even if you are going to ignore it (for now). If you plan to rethrow the Exception, use
@ -347,7 +347,7 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
}
}
}
/**
* Call this method in place of any situation where you would normally (re)throw an exception
* (already passed to the {@link #caught} method because maxErrors was exceeded
@ -374,9 +374,9 @@ public class TolerantUpdateProcessor extends UpdateRequestProcessor {
if (null == first) {
return; // no exception to annotate
}
assert null != errors : "how do we have an exception to annotate w/o any errors?";
NamedList<String> firstErrMetadata = first.getMetadata();
if (null == firstErrMetadata) { // obnoxious
firstErrMetadata = new NamedList<String>();

114
solr/core/src/java/org/apache/solr/util/stats/InstrumentedHttpListenerFactory.java Normal file
View File

@ -0,0 +1,114 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.util.stats;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import org.apache.solr.client.solrj.impl.HttpListenerFactory;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.eclipse.jetty.client.api.Request;
import org.eclipse.jetty.client.api.Result;
import static org.apache.solr.metrics.SolrMetricManager.mkName;
/**
* A HttpListenerFactory tracks metrics interesting to solr
* Inspired and partially copied from dropwizard httpclient library
*/
public class InstrumentedHttpListenerFactory implements SolrMetricProducer, HttpListenerFactory {
public interface NameStrategy {
String getNameFor(String scope, Request request);
}
private static final NameStrategy QUERYLESS_URL_AND_METHOD =
(scope, request) -> {
String schemeHostPort = request.getScheme() + "://" + request.getHost() + ":" + request.getPort() + request.getPath();
return mkName(schemeHostPort + "." + methodNameString(request), scope);
};
private static final NameStrategy METHOD_ONLY =
(scope, request) -> mkName(methodNameString(request), scope);
private static final NameStrategy HOST_AND_METHOD =
(scope, request) -> {
String schemeHostPort = request.getScheme() + "://" + request.getHost() + ":" + request.getPort();
return mkName(schemeHostPort + "." + methodNameString(request), scope);
};
public static final Map<String, NameStrategy> KNOWN_METRIC_NAME_STRATEGIES = new HashMap<>(3);
static {
KNOWN_METRIC_NAME_STRATEGIES.put("queryLessURLAndMethod", QUERYLESS_URL_AND_METHOD);
KNOWN_METRIC_NAME_STRATEGIES.put("hostAndMethod", HOST_AND_METHOD);
KNOWN_METRIC_NAME_STRATEGIES.put("methodOnly", METHOD_ONLY);
}
protected MetricRegistry metricsRegistry;
protected SolrMetricManager metricManager;
protected String registryName;
protected String scope;
protected NameStrategy nameStrategy;
public InstrumentedHttpListenerFactory(NameStrategy nameStrategy) {
this.nameStrategy = nameStrategy;
}
private static String methodNameString(Request request) {
return request.getMethod().toLowerCase(Locale.ROOT) + ".requests";
}
@Override
public RequestResponseListener get() {
return new RequestResponseListener() {
Timer.Context timerContext;
@Override
public void onBegin(Request request) {
if (metricsRegistry != null) {
timerContext = timer(request).time();
}
}
@Override
public void onComplete(Result result) {
if (timerContext != null) {
timerContext.stop();
}
}
};
}
private Timer timer(Request request) {
return metricsRegistry.timer(nameStrategy.getNameFor(scope, request));
}
@Override
public void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) {
this.metricManager = manager;
this.registryName = registry;
this.metricsRegistry = manager.registry(registry);
this.scope = scope;
}
}

1
solr/core/src/test-files/log4j2.xml
View File

@ -31,6 +31,7 @@
<Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
<Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
<AppenderRef ref="STDERR"/>

4
solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java
View File

@ -40,6 +40,7 @@ import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.cloud.Overseer.LeaderStatus;
import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;
import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler;
@ -250,7 +251,8 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
protected Set<String> commonMocks(int liveNodesCount) throws Exception {
when(shardHandlerFactoryMock.getShardHandler()).thenReturn(shardHandlerMock);
when(shardHandlerFactoryMock.getShardHandler(any())).thenReturn(shardHandlerMock);
when(shardHandlerFactoryMock.getShardHandler(any(Http2SolrClient.class))).thenReturn(shardHandlerMock);
when(shardHandlerFactoryMock.getShardHandler(any(HttpClient.class))).thenReturn(shardHandlerMock);
when(workQueueMock.peekTopN(anyInt(), any(), anyLong())).thenAnswer(invocation -> {
Object result;
int count = 0;

2
solr/core/src/test/org/apache/solr/cloud/SSLMigrationTest.java
View File

@ -80,7 +80,7 @@ public class SSLMigrationTest extends AbstractFullDistribZkTestBase {
.stopAtShutdown(false)
.withServlets(getExtraServlets())
.withFilters(getExtraRequestFilters())
.withSSLConfig(sslConfig)
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
Properties props = new Properties();

15
solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
View File

@ -41,6 +41,7 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@ -90,11 +91,13 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
// undo the randomization of our super class
log.info("NOTE: This Test ignores the randomized SSL & clientAuth settings selected by base class");
HttpClientUtil.resetHttpClientBuilder(); // also resets SchemaRegistryProvider
Http2SolrClient.resetSslContextFactory();
System.clearProperty(ZkStateReader.URL_SCHEME);
}
@After
public void after() {
HttpClientUtil.resetHttpClientBuilder(); // also resets SchemaRegistryProvider
Http2SolrClient.resetSslContextFactory();
System.clearProperty(ZkStateReader.URL_SCHEME);
SSLContext.setDefault(DEFAULT_SSL_CONTEXT);
}
@ -102,6 +105,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
public void testNoSsl() throws Exception {
final SSLTestConfig sslConfig = new SSLTestConfig(false, false);
HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
Http2SolrClient.setDefaultSSLConfig(sslConfig.buildClientSSLConfig());
System.setProperty(ZkStateReader.URL_SCHEME, "http");
checkClusterWithNodeReplacement(sslConfig);
}
@ -112,6 +116,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
// options.
final SSLTestConfig sslConfig = new SSLTestConfig(false, true);
HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
Http2SolrClient.setDefaultSSLConfig(sslConfig.buildClientSSLConfig());
System.setProperty(ZkStateReader.URL_SCHEME, "http");
checkClusterWithNodeReplacement(sslConfig);
}
@ -119,6 +124,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
public void testSslAndNoClientAuth() throws Exception {
final SSLTestConfig sslConfig = new SSLTestConfig(true, false);
HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
Http2SolrClient.setDefaultSSLConfig(sslConfig.buildClientSSLConfig());
System.setProperty(ZkStateReader.URL_SCHEME, "https");
checkClusterWithNodeReplacement(sslConfig);
}
@ -129,6 +135,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
final SSLTestConfig sslConfig = new SSLTestConfig(true, true);
HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
Http2SolrClient.setDefaultSSLConfig(sslConfig.buildClientSSLConfig());
System.setProperty(ZkStateReader.URL_SCHEME, "https");
checkClusterWithNodeReplacement(sslConfig);
}
@ -137,6 +144,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
public void testSslWithCheckPeerName() throws Exception {
final SSLTestConfig sslConfig = new SSLTestConfig(true, false, true);
HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
Http2SolrClient.setDefaultSSLConfig(sslConfig.buildClientSSLConfig());
System.setProperty(ZkStateReader.URL_SCHEME, "https");
checkClusterWithNodeReplacement(sslConfig);
}
@ -153,7 +161,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
*/
private void checkClusterWithNodeReplacement(SSLTestConfig sslConfig) throws Exception {
final JettyConfig config = JettyConfig.builder().withSSLConfig(sslConfig).build();
final JettyConfig config = JettyConfig.builder().withSSLConfig(sslConfig.buildServerSSLConfig()).build();
final MiniSolrCloudCluster cluster = new MiniSolrCloudCluster(NUM_SERVERS, createTempDir(), config);
try {
checkClusterWithCollectionCreations(cluster, sslConfig);
@ -166,6 +174,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
System.setProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME,
Boolean.toString(sslConfig.getCheckPeerName()));
HttpClientUtil.resetHttpClientBuilder();
Http2SolrClient.resetSslContextFactory();
// recheck that we can communicate with all the jetty instances in our cluster
checkClusterJettys(cluster, sslConfig);
@ -180,8 +189,9 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
// certs with a bogus hostname/ip and clients shouldn't care...
final SSLTestConfig sslConfig = new SSLTestConfig(true, false, false);
HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
Http2SolrClient.setDefaultSSLConfig(sslConfig.buildClientSSLConfig());
System.setProperty(ZkStateReader.URL_SCHEME, "https");
final JettyConfig config = JettyConfig.builder().withSSLConfig(sslConfig).build();
final JettyConfig config = JettyConfig.builder().withSSLConfig(sslConfig.buildServerSSLConfig()).build();
final MiniSolrCloudCluster cluster = new MiniSolrCloudCluster(NUM_SERVERS, createTempDir(), config);
try {
checkClusterWithCollectionCreations(cluster, sslConfig);
@ -190,6 +200,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
// our existing certificate, but *does* care about validating the peer name
System.setProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME, "true");
HttpClientUtil.resetHttpClientBuilder();
Http2SolrClient.resetSslContextFactory();
// and validate we get failures when trying to talk to our cluster...
final List<JettySolrRunner> jettys = cluster.getJettySolrRunners();

4
solr/core/src/test/org/apache/solr/handler/component/TestHttpShardHandlerFactory.java
View File

@ -22,8 +22,8 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.impl.LBSolrClient;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ShardParams;
@ -80,7 +80,7 @@ public class TestHttpShardHandlerFactory extends SolrTestCaseJ4 {
}
// create LBHttpSolrClient request
final LBHttpSolrClient.Req req = httpShardHandlerFactory.newLBHttpSolrClientReq(queryRequest, urls);
final LBSolrClient.Req req = httpShardHandlerFactory.newLBHttpSolrClientReq(queryRequest, urls);
// actual vs. expected test
final int actualNumServersToTry = req.getNumServersToTry().intValue();

5
solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java
View File

@ -233,6 +233,11 @@ public class BasicAuthIntegrationTest extends SolrCloudAuthTestCase {
executeCommand(baseUrl + authzPrefix, cl,"{set-permission : { name : update , role : admin}}", "harry", "HarryIsUberCool");
UpdateRequest del = new UpdateRequest().deleteByQuery("*:*");
del.setBasicAuthCredentials("harry","HarryIsUberCool");
del.setCommitWithin(10);
del.process(cluster.getSolrClient(), COLLECTION);
addDocument("harry","HarryIsUberCool","id", "4");
executeCommand(baseUrl + authcPrefix, cl, "{set-property : { blockUnknown: true}}", "harry", "HarryIsUberCool");

64
solr/core/src/test/org/apache/solr/security/HttpParamDelegationTokenPlugin.java
View File

@ -24,6 +24,7 @@ import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import javax.servlet.Filter;
@ -49,12 +50,15 @@ import org.apache.http.HttpRequestInterceptor;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.protocol.HttpContext;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpListenerFactory;
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.request.SolrRequestInfo;
import org.eclipse.jetty.client.api.Request;
/**
* AuthenticationHandler that supports delegation tokens and simple
@ -73,27 +77,7 @@ public class HttpParamDelegationTokenPlugin extends KerberosPlugin {
private final HttpRequestInterceptor interceptor = new HttpRequestInterceptor() {
@Override
public void process(HttpRequest httpRequest, HttpContext httpContext) throws HttpException, IOException {
SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo();
String usr;
if (reqInfo != null) {
Principal principal = reqInfo.getReq().getUserPrincipal();
if (principal == null) {
//this had a request but not authenticated
//so we don't not need to set a principal
return;
} else {
usr = principal.getName();
}
} else {
if (!isSolrThread()) {
//if this is not running inside a Solr threadpool (as in testcases)
// then no need to add any header
return;
}
//this request seems to be originated from Solr itself
usr = "$"; //special name to denote the user is the node itself
}
httpRequest.setHeader(INTERNAL_REQUEST_HEADER, usr);
getPrincipal().ifPresent(usr -> httpRequest.setHeader(INTERNAL_REQUEST_HEADER, usr));
}
};
@ -139,10 +123,46 @@ public class HttpParamDelegationTokenPlugin extends KerberosPlugin {
}
}
private Optional<String> getPrincipal() {
SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo();
String usr;
if (reqInfo != null) {
Principal principal = reqInfo.getReq().getUserPrincipal();
if (principal == null) {
//this had a request but not authenticated
//so we don't not need to set a principal
return Optional.empty();
} else {
usr = principal.getName();
}
} else {
if (!isSolrThread()) {
//if this is not running inside a Solr threadpool (as in testcases)
// then no need to add any header
return Optional.empty();
}
//this request seems to be originated from Solr itself
usr = "$"; //special name to denote the user is the node itself
}
return Optional.of(usr);
}
@Override
public void setup(Http2SolrClient client) {
final HttpListenerFactory.RequestResponseListener listener = new HttpListenerFactory.RequestResponseListener() {
@Override
public void onQueued(Request request) {
getPrincipal().ifPresent(usr -> request.header(INTERNAL_REQUEST_HEADER, usr));
}
};
client.addListenerFactory(() -> listener);
}
@Override
public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) {
HttpClientUtil.addRequestInterceptor(interceptor);
return super.getHttpClientBuilder(builder);
builder = super.getHttpClientBuilder(builder);
return builder;
}
@Override

156
solr/core/src/test/org/apache/solr/update/MockingHttp2SolrClient.java Normal file
View File

@ -0,0 +1,156 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.update;
import java.io.IOException;
import java.net.ConnectException;
import java.net.SocketException;
import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
public class MockingHttp2SolrClient extends Http2SolrClient {
public enum Exp {CONNECT_EXCEPTION, SOCKET_EXCEPTION, BAD_REQUEST};
private volatile Exp exp = null;
private boolean oneExpPerReq;
private Set<SolrRequest> reqGotException;
public MockingHttp2SolrClient(String baseSolrUrl, Builder builder) {
super(baseSolrUrl, builder);
this.oneExpPerReq = builder.oneExpPerReq;
this.reqGotException = new HashSet<>();
}
public static class Builder extends Http2SolrClient.Builder {
private boolean oneExpPerReq = false;
public Builder(UpdateShardHandlerConfig config) {
super();
this.connectionTimeout(config.getDistributedConnectionTimeout());
this.idleTimeout(config.getDistributedSocketTimeout());
}
public MockingHttp2SolrClient build() {
return new MockingHttp2SolrClient(null, this);
}
// DBQ won't cause exception
Builder oneExpPerReq() {
oneExpPerReq = true;
return this;
}
}
public void setExp(Exp exp) {
this.exp = exp;
}
private Exception exception() {
switch (exp) {
case CONNECT_EXCEPTION:
return new ConnectException();
case SOCKET_EXCEPTION:
return new SocketException();
case BAD_REQUEST:
return new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Bad Request");
default:
break;
}
return null;
}
@Override
public NamedList<Object> request(SolrRequest request, String collection)
throws SolrServerException, IOException {
if (request instanceof UpdateRequest) {
UpdateRequest ur = (UpdateRequest) request;
if (!ur.getDeleteQuery().isEmpty())
return super.request(request, collection);
}
if (exp != null) {
if (oneExpPerReq) {
if (reqGotException.contains(request))
return super.request(request, collection);
else
reqGotException.add(request);
}
Exception e = exception();
if (e instanceof IOException) {
if (LuceneTestCase.random().nextBoolean()) {
throw (IOException) e;
} else {
throw new SolrServerException(e);
}
} else if (e instanceof SolrServerException) {
throw (SolrServerException) e;
} else {
throw new SolrServerException(e);
}
}
return super.request(request, collection);
}
public NamedList<Object> request(SolrRequest request, String collection, OnComplete onComplete)
throws SolrServerException, IOException {
if (request instanceof UpdateRequest) {
UpdateRequest ur = (UpdateRequest) request;
// won't throw exception if request is DBQ
if (ur.getDeleteQuery() != null && !ur.getDeleteQuery().isEmpty()) {
return super.request(request, collection, onComplete);
}
}
if (exp != null) {
if (oneExpPerReq) {
if (reqGotException.contains(request)) {
return super.request(request, collection, onComplete);
}
else
reqGotException.add(request);
}
Exception e = exception();
if (e instanceof IOException) {
if (LuceneTestCase.random().nextBoolean()) {
throw (IOException) e;
} else {
throw new SolrServerException(e);
}
} else if (e instanceof SolrServerException) {
throw (SolrServerException) e;
} else {
throw new SolrServerException(e);
}
}
return super.request(request, collection, onComplete);
}
}

1
solr/licenses/http2-client-9.4.14.v20181114.jar.sha1 Normal file
View File

@ -0,0 +1 @@
78917d06b788fad75fdd4fa73d8d8ff9679200dd

202
solr/licenses/http2-client-LICENSE-ASL.txt Normal file
View File

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

111
solr/licenses/http2-client-NOTICE.txt Normal file
View File

@ -0,0 +1,111 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN">
<html><head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>Eclipse.org Software User Agreement</title>
</head><body lang="EN-US" link="blue" vlink="purple">
<h2>Eclipse Foundation Software User Agreement</h2>
<p>March 17, 2005</p>
<h3>Usage Of Content</h3>
<p>THE ECLIPSE FOUNDATION MAKES AVAILABLE SOFTWARE, DOCUMENTATION, INFORMATION AND/OR OTHER MATERIALS FOR OPEN SOURCE PROJECTS
(COLLECTIVELY "CONTENT"). USE OF THE CONTENT IS GOVERNED BY THE TERMS AND CONDITIONS OF THIS AGREEMENT AND/OR THE TERMS AND
CONDITIONS OF LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. BY USING THE CONTENT, YOU AGREE THAT YOUR USE
OF THE CONTENT IS GOVERNED BY THIS AGREEMENT AND/OR THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR
NOTICES INDICATED OR REFERENCED BELOW. IF YOU DO NOT AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT AND THE TERMS AND
CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW, THEN YOU MAY NOT USE THE CONTENT.</p>
<h3>Applicable Licenses</h3>
<p>Unless otherwise indicated, all Content made available by the
Eclipse Foundation is provided to you under the terms and conditions of
the Eclipse Public License Version 1.0 ("EPL"). A copy of the EPL is
provided with this Content and is also available at <a href="http://www.eclipse.org/legal/epl-v10.html">http://www.eclipse.org/legal/epl-v10.html</a>.
For purposes of the EPL, "Program" will mean the Content.</p>
<p>Content includes, but is not limited to, source code, object code,
documentation and other files maintained in the Eclipse.org CVS
repository ("Repository") in CVS modules ("Modules") and made available
as downloadable archives ("Downloads").</p>
<ul>
<li>Content may be structured and packaged into modules to
facilitate delivering, extending, and upgrading the Content. Typical
modules may include plug-ins ("Plug-ins"), plug-in fragments
("Fragments"), and features ("Features").</li>
<li>Each Plug-in or Fragment may be packaged as a sub-directory or JAR (Java&#8482; ARchive) in a directory named "plugins".</li>
<li>A
Feature is a bundle of one or more Plug-ins and/or Fragments and
associated material. Each Feature may be packaged as a sub-directory in
a directory named "features". Within a Feature, files named
"feature.xml" may contain a list of the names and version numbers of
the Plug-ins and/or Fragments associated with that Feature.</li>
<li>Features
may also include other Features ("Included Features"). Within a
Feature, files named "feature.xml" may contain a list of the names and
version numbers of Included Features.</li>
</ul>
<p>The terms and conditions governing Plug-ins and Fragments should be
contained in files named "about.html" ("Abouts"). The terms and
conditions governing Features and
Included Features should be contained in files named "license.html"
("Feature Licenses"). Abouts and Feature Licenses may be located in any
directory of a Download or Module
including, but not limited to the following locations:</p>
<ul>
<li>The top-level (root) directory</li>
<li>Plug-in and Fragment directories</li>
<li>Inside Plug-ins and Fragments packaged as JARs</li>
<li>Sub-directories of the directory named "src" of certain Plug-ins</li>
<li>Feature directories</li>
</ul>
<p>Note: if a Feature made available by the Eclipse Foundation is
installed using the Eclipse Update Manager, you must agree to a license
("Feature Update License") during the
installation process. If the Feature contains Included Features, the
Feature Update License should either provide you with the terms and
conditions governing the Included Features or
inform you where you can locate them. Feature Update Licenses may be
found in the "license" property of files named "feature.properties"
found within a Feature.
Such Abouts, Feature Licenses, and Feature Update Licenses contain the
terms and conditions (or references to such terms and conditions) that
govern your use of the associated Content in
that directory.</p>
<p>THE ABOUTS, FEATURE LICENSES, AND FEATURE UPDATE LICENSES MAY REFER
TO THE EPL OR OTHER LICENSE AGREEMENTS, NOTICES OR TERMS AND
CONDITIONS. SOME OF THESE
OTHER LICENSE AGREEMENTS MAY INCLUDE (BUT ARE NOT LIMITED TO):</p>
<ul>
<li>Common Public License Version 1.0 (available at <a href="http://www.eclipse.org/legal/cpl-v10.html">http://www.eclipse.org/legal/cpl-v10.html</a>)</li>
<li>Apache Software License 1.1 (available at <a href="http://www.apache.org/licenses/LICENSE">http://www.apache.org/licenses/LICENSE</a>)</li>
<li>Apache Software License 2.0 (available at <a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a>)</li>
<li>IBM Public License 1.0 (available at <a href="http://oss.software.ibm.com/developerworks/opensource/license10.html">http://oss.software.ibm.com/developerworks/opensource/license10.html</a>)</li>
<li>Metro Link Public License 1.00 (available at <a href="http://www.opengroup.org/openmotif/supporters/metrolink/license.html">http://www.opengroup.org/openmotif/supporters/metrolink/license.html</a>)</li>
<li>Mozilla Public License Version 1.1 (available at <a href="http://www.mozilla.org/MPL/MPL-1.1.html">http://www.mozilla.org/MPL/MPL-1.1.html</a>)</li>
</ul>
<p>IT IS YOUR OBLIGATION TO READ AND ACCEPT ALL SUCH TERMS AND
CONDITIONS PRIOR TO USE OF THE CONTENT. If no About, Feature License,
or Feature Update License is provided, please
contact the Eclipse Foundation to determine what terms and conditions
govern that particular Content.</p>
<h3>Cryptography</h3>
<p>Content may contain encryption software. The country in which you
are currently may have restrictions on the import, possession, and use,
and/or re-export to another country, of encryption software. BEFORE
using any encryption software, please check the country's laws,
regulations and policies concerning the import, possession, or use, and
re-export of encryption software, to see if this is permitted.</p>
<small>Java and all Java-based trademarks are trademarks of Sun Microsystems, Inc. in the United States, other countries, or both.</small>
</body></html>

1
solr/licenses/http2-common-9.4.14.v20181114.jar.sha1 Normal file
View File

@ -0,0 +1 @@
820ca2201ad531983cc3a8e2a82153268828d025

202
solr/licenses/http2-common-LICENSE-ASL.txt Normal file
View File

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

111
solr/licenses/http2-common-NOTICE.txt Normal file
View File

@ -0,0 +1,111 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN">
<html><head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>Eclipse.org Software User Agreement</title>
</head><body lang="EN-US" link="blue" vlink="purple">
<h2>Eclipse Foundation Software User Agreement</h2>
<p>March 17, 2005</p>
<h3>Usage Of Content</h3>
<p>THE ECLIPSE FOUNDATION MAKES AVAILABLE SOFTWARE, DOCUMENTATION, INFORMATION AND/OR OTHER MATERIALS FOR OPEN SOURCE PROJECTS
(COLLECTIVELY "CONTENT"). USE OF THE CONTENT IS GOVERNED BY THE TERMS AND CONDITIONS OF THIS AGREEMENT AND/OR THE TERMS AND
CONDITIONS OF LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. BY USING THE CONTENT, YOU AGREE THAT YOUR USE
OF THE CONTENT IS GOVERNED BY THIS AGREEMENT AND/OR THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR
NOTICES INDICATED OR REFERENCED BELOW. IF YOU DO NOT AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT AND THE TERMS AND
CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW, THEN YOU MAY NOT USE THE CONTENT.</p>
<h3>Applicable Licenses</h3>
<p>Unless otherwise indicated, all Content made available by the
Eclipse Foundation is provided to you under the terms and conditions of
the Eclipse Public License Version 1.0 ("EPL"). A copy of the EPL is
provided with this Content and is also available at <a href="http://www.eclipse.org/legal/epl-v10.html">http://www.eclipse.org/legal/epl-v10.html</a>.
For purposes of the EPL, "Program" will mean the Content.</p>
<p>Content includes, but is not limited to, source code, object code,
documentation and other files maintained in the Eclipse.org CVS
repository ("Repository") in CVS modules ("Modules") and made available
as downloadable archives ("Downloads").</p>
<ul>
<li>Content may be structured and packaged into modules to
facilitate delivering, extending, and upgrading the Content. Typical
modules may include plug-ins ("Plug-ins"), plug-in fragments
("Fragments"), and features ("Features").</li>
<li>Each Plug-in or Fragment may be packaged as a sub-directory or JAR (Java&#8482; ARchive) in a directory named "plugins".</li>
<li>A
Feature is a bundle of one or more Plug-ins and/or Fragments and
associated material. Each Feature may be packaged as a sub-directory in
a directory named "features". Within a Feature, files named
"feature.xml" may contain a list of the names and version numbers of
the Plug-ins and/or Fragments associated with that Feature.</li>
<li>Features
may also include other Features ("Included Features"). Within a
Feature, files named "feature.xml" may contain a list of the names and
version numbers of Included Features.</li>
</ul>
<p>The terms and conditions governing Plug-ins and Fragments should be
contained in files named "about.html" ("Abouts"). The terms and
conditions governing Features and
Included Features should be contained in files named "license.html"
("Feature Licenses"). Abouts and Feature Licenses may be located in any
directory of a Download or Module
including, but not limited to the following locations:</p>
<ul>
<li>The top-level (root) directory</li>
<li>Plug-in and Fragment directories</li>
<li>Inside Plug-ins and Fragments packaged as JARs</li>
<li>Sub-directories of the directory named "src" of certain Plug-ins</li>
<li>Feature directories</li>
</ul>
<p>Note: if a Feature made available by the Eclipse Foundation is
installed using the Eclipse Update Manager, you must agree to a license
("Feature Update License") during the
installation process. If the Feature contains Included Features, the
Feature Update License should either provide you with the terms and
conditions governing the Included Features or
inform you where you can locate them. Feature Update Licenses may be
found in the "license" property of files named "feature.properties"
found within a Feature.
Such Abouts, Feature Licenses, and Feature Update Licenses contain the
terms and conditions (or references to such terms and conditions) that
govern your use of the associated Content in
that directory.</p>
<p>THE ABOUTS, FEATURE LICENSES, AND FEATURE UPDATE LICENSES MAY REFER
TO THE EPL OR OTHER LICENSE AGREEMENTS, NOTICES OR TERMS AND
CONDITIONS. SOME OF THESE
OTHER LICENSE AGREEMENTS MAY INCLUDE (BUT ARE NOT LIMITED TO):</p>
<ul>
<li>Common Public License Version 1.0 (available at <a href="http://www.eclipse.org/legal/cpl-v10.html">http://www.eclipse.org/legal/cpl-v10.html</a>)</li>
<li>Apache Software License 1.1 (available at <a href="http://www.apache.org/licenses/LICENSE">http://www.apache.org/licenses/LICENSE</a>)</li>
<li>Apache Software License 2.0 (available at <a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a>)</li>
<li>IBM Public License 1.0 (available at <a href="http://oss.software.ibm.com/developerworks/opensource/license10.html">http://oss.software.ibm.com/developerworks/opensource/license10.html</a>)</li>
<li>Metro Link Public License 1.00 (available at <a href="http://www.opengroup.org/openmotif/supporters/metrolink/license.html">http://www.opengroup.org/openmotif/supporters/metrolink/license.html</a>)</li>
<li>Mozilla Public License Version 1.1 (available at <a href="http://www.mozilla.org/MPL/MPL-1.1.html">http://www.mozilla.org/MPL/MPL-1.1.html</a>)</li>
</ul>
<p>IT IS YOUR OBLIGATION TO READ AND ACCEPT ALL SUCH TERMS AND
CONDITIONS PRIOR TO USE OF THE CONTENT. If no About, Feature License,
or Feature Update License is provided, please
contact the Eclipse Foundation to determine what terms and conditions
govern that particular Content.</p>
<h3>Cryptography</h3>
<p>Content may contain encryption software. The country in which you
are currently may have restrictions on the import, possession, and use,
and/or re-export to another country, of encryption software. BEFORE
using any encryption software, please check the country's laws,
regulations and policies concerning the import, possession, or use, and
re-export of encryption software, to see if this is permitted.</p>
<small>Java and all Java-based trademarks are trademarks of Sun Microsystems, Inc. in the United States, other countries, or both.</small>
</body></html>

1
solr/licenses/http2-hpack-9.4.14.v20181114.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e6cc7ae5b5749afe8b787595b28c6813c13c3ac2

202
solr/licenses/http2-hpack-LICENSE-ASL.txt Normal file
View File

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

111
solr/licenses/http2-hpack-NOTICE.txt Normal file
View File

@ -0,0 +1,111 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN">
<html><head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>Eclipse.org Software User Agreement</title>
</head><body lang="EN-US" link="blue" vlink="purple">
<h2>Eclipse Foundation Software User Agreement</h2>
<p>March 17, 2005</p>
<h3>Usage Of Content</h3>
<p>THE ECLIPSE FOUNDATION MAKES AVAILABLE SOFTWARE, DOCUMENTATION, INFORMATION AND/OR OTHER MATERIALS FOR OPEN SOURCE PROJECTS
(COLLECTIVELY "CONTENT"). USE OF THE CONTENT IS GOVERNED BY THE TERMS AND CONDITIONS OF THIS AGREEMENT AND/OR THE TERMS AND
CONDITIONS OF LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. BY USING THE CONTENT, YOU AGREE THAT YOUR USE
OF THE CONTENT IS GOVERNED BY THIS AGREEMENT AND/OR THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR
NOTICES INDICATED OR REFERENCED BELOW. IF YOU DO NOT AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT AND THE TERMS AND
CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW, THEN YOU MAY NOT USE THE CONTENT.</p>
<h3>Applicable Licenses</h3>
<p>Unless otherwise indicated, all Content made available by the
Eclipse Foundation is provided to you under the terms and conditions of
the Eclipse Public License Version 1.0 ("EPL"). A copy of the EPL is
provided with this Content and is also available at <a href="http://www.eclipse.org/legal/epl-v10.html">http://www.eclipse.org/legal/epl-v10.html</a>.
For purposes of the EPL, "Program" will mean the Content.</p>
<p>Content includes, but is not limited to, source code, object code,
documentation and other files maintained in the Eclipse.org CVS
repository ("Repository") in CVS modules ("Modules") and made available
as downloadable archives ("Downloads").</p>
<ul>
<li>Content may be structured and packaged into modules to
facilitate delivering, extending, and upgrading the Content. Typical
modules may include plug-ins ("Plug-ins"), plug-in fragments
("Fragments"), and features ("Features").</li>
<li>Each Plug-in or Fragment may be packaged as a sub-directory or JAR (Java&#8482; ARchive) in a directory named "plugins".</li>
<li>A
Feature is a bundle of one or more Plug-ins and/or Fragments and
associated material. Each Feature may be packaged as a sub-directory in
a directory named "features". Within a Feature, files named
"feature.xml" may contain a list of the names and version numbers of
the Plug-ins and/or Fragments associated with that Feature.</li>
<li>Features
may also include other Features ("Included Features"). Within a
Feature, files named "feature.xml" may contain a list of the names and
version numbers of Included Features.</li>
</ul>
<p>The terms and conditions governing Plug-ins and Fragments should be
contained in files named "about.html" ("Abouts"). The terms and
conditions governing Features and
Included Features should be contained in files named "license.html"
("Feature Licenses"). Abouts and Feature Licenses may be located in any
directory of a Download or Module
including, but not limited to the following locations:</p>
<ul>
<li>The top-level (root) directory</li>
<li>Plug-in and Fragment directories</li>
<li>Inside Plug-ins and Fragments packaged as JARs</li>
<li>Sub-directories of the directory named "src" of certain Plug-ins</li>
<li>Feature directories</li>
</ul>
<p>Note: if a Feature made available by the Eclipse Foundation is
installed using the Eclipse Update Manager, you must agree to a license
("Feature Update License") during the
installation process. If the Feature contains Included Features, the
Feature Update License should either provide you with the terms and
conditions governing the Included Features or
inform you where you can locate them. Feature Update Licenses may be
found in the "license" property of files named "feature.properties"
found within a Feature.
Such Abouts, Feature Licenses, and Feature Update Licenses contain the
terms and conditions (or references to such terms and conditions) that
govern your use of the associated Content in
that directory.</p>
<p>THE ABOUTS, FEATURE LICENSES, AND FEATURE UPDATE LICENSES MAY REFER
TO THE EPL OR OTHER LICENSE AGREEMENTS, NOTICES OR TERMS AND
CONDITIONS. SOME OF THESE
OTHER LICENSE AGREEMENTS MAY INCLUDE (BUT ARE NOT LIMITED TO):</p>
<ul>
<li>Common Public License Version 1.0 (available at <a href="http://www.eclipse.org/legal/cpl-v10.html">http://www.eclipse.org/legal/cpl-v10.html</a>)</li>
<li>Apache Software License 1.1 (available at <a href="http://www.apache.org/licenses/LICENSE">http://www.apache.org/licenses/LICENSE</a>)</li>
<li>Apache Software License 2.0 (available at <a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a>)</li>
<li>IBM Public License 1.0 (available at <a href="http://oss.software.ibm.com/developerworks/opensource/license10.html">http://oss.software.ibm.com/developerworks/opensource/license10.html</a>)</li>
<li>Metro Link Public License 1.00 (available at <a href="http://www.opengroup.org/openmotif/supporters/metrolink/license.html">http://www.opengroup.org/openmotif/supporters/metrolink/license.html</a>)</li>
<li>Mozilla Public License Version 1.1 (available at <a href="http://www.mozilla.org/MPL/MPL-1.1.html">http://www.mozilla.org/MPL/MPL-1.1.html</a>)</li>
</ul>
<p>IT IS YOUR OBLIGATION TO READ AND ACCEPT ALL SUCH TERMS AND
CONDITIONS PRIOR TO USE OF THE CONTENT. If no About, Feature License,
or Feature Update License is provided, please
contact the Eclipse Foundation to determine what terms and conditions
govern that particular Content.</p>
<h3>Cryptography</h3>
<p>Content may contain encryption software. The country in which you
are currently may have restrictions on the import, possession, and use,
and/or re-export to another country, of encryption software. BEFORE
using any encryption software, please check the country's laws,
regulations and policies concerning the import, possession, or use, and
re-export of encryption software, to see if this is permitted.</p>
<small>Java and all Java-based trademarks are trademarks of Sun Microsystems, Inc. in the United States, other countries, or both.</small>
</body></html>

1
solr/licenses/http2-http-client-transport-9.4.14.v20181114.jar.sha1 Normal file
View File

@ -0,0 +1 @@
77139eb205d3ddb2d19458c534c734f11491a429

202
solr/licenses/http2-http-client-transport-LICENSE-ASL.txt Normal file
View File

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

111
solr/licenses/http2-http-client-transport-NOTICE.txt Normal file
View File

@ -0,0 +1,111 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN">
<html><head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>Eclipse.org Software User Agreement</title>
</head><body lang="EN-US" link="blue" vlink="purple">
<h2>Eclipse Foundation Software User Agreement</h2>
<p>March 17, 2005</p>
<h3>Usage Of Content</h3>
<p>THE ECLIPSE FOUNDATION MAKES AVAILABLE SOFTWARE, DOCUMENTATION, INFORMATION AND/OR OTHER MATERIALS FOR OPEN SOURCE PROJECTS
(COLLECTIVELY "CONTENT"). USE OF THE CONTENT IS GOVERNED BY THE TERMS AND CONDITIONS OF THIS AGREEMENT AND/OR THE TERMS AND
CONDITIONS OF LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. BY USING THE CONTENT, YOU AGREE THAT YOUR USE
OF THE CONTENT IS GOVERNED BY THIS AGREEMENT AND/OR THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR
NOTICES INDICATED OR REFERENCED BELOW. IF YOU DO NOT AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT AND THE TERMS AND
CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW, THEN YOU MAY NOT USE THE CONTENT.</p>
<h3>Applicable Licenses</h3>
<p>Unless otherwise indicated, all Content made available by the
Eclipse Foundation is provided to you under the terms and conditions of
the Eclipse Public License Version 1.0 ("EPL"). A copy of the EPL is
provided with this Content and is also available at <a href="http://www.eclipse.org/legal/epl-v10.html">http://www.eclipse.org/legal/epl-v10.html</a>.
For purposes of the EPL, "Program" will mean the Content.</p>
<p>Content includes, but is not limited to, source code, object code,
documentation and other files maintained in the Eclipse.org CVS
repository ("Repository") in CVS modules ("Modules") and made available
as downloadable archives ("Downloads").</p>
<ul>
<li>Content may be structured and packaged into modules to
facilitate delivering, extending, and upgrading the Content. Typical
modules may include plug-ins ("Plug-ins"), plug-in fragments
("Fragments"), and features ("Features").</li>
<li>Each Plug-in or Fragment may be packaged as a sub-directory or JAR (Java&#8482; ARchive) in a directory named "plugins".</li>
<li>A
Feature is a bundle of one or more Plug-ins and/or Fragments and
associated material. Each Feature may be packaged as a sub-directory in
a directory named "features". Within a Feature, files named
"feature.xml" may contain a list of the names and version numbers of
the Plug-ins and/or Fragments associated with that Feature.</li>
<li>Features
may also include other Features ("Included Features"). Within a
Feature, files named "feature.xml" may contain a list of the names and
version numbers of Included Features.</li>
</ul>
<p>The terms and conditions governing Plug-ins and Fragments should be
contained in files named "about.html" ("Abouts"). The terms and
conditions governing Features and
Included Features should be contained in files named "license.html"
("Feature Licenses"). Abouts and Feature Licenses may be located in any
directory of a Download or Module
including, but not limited to the following locations:</p>
<ul>
<li>The top-level (root) directory</li>
<li>Plug-in and Fragment directories</li>
<li>Inside Plug-ins and Fragments packaged as JARs</li>
<li>Sub-directories of the directory named "src" of certain Plug-ins</li>
<li>Feature directories</li>
</ul>
<p>Note: if a Feature made available by the Eclipse Foundation is
installed using the Eclipse Update Manager, you must agree to a license
("Feature Update License") during the
installation process. If the Feature contains Included Features, the
Feature Update License should either provide you with the terms and
conditions governing the Included Features or
inform you where you can locate them. Feature Update Licenses may be
found in the "license" property of files named "feature.properties"
found within a Feature.
Such Abouts, Feature Licenses, and Feature Update Licenses contain the
terms and conditions (or references to such terms and conditions) that
govern your use of the associated Content in
that directory.</p>
<p>THE ABOUTS, FEATURE LICENSES, AND FEATURE UPDATE LICENSES MAY REFER
TO THE EPL OR OTHER LICENSE AGREEMENTS, NOTICES OR TERMS AND
CONDITIONS. SOME OF THESE
OTHER LICENSE AGREEMENTS MAY INCLUDE (BUT ARE NOT LIMITED TO):</p>
<ul>
<li>Common Public License Version 1.0 (available at <a href="http://www.eclipse.org/legal/cpl-v10.html">http://www.eclipse.org/legal/cpl-v10.html</a>)</li>
<li>Apache Software License 1.1 (available at <a href="http://www.apache.org/licenses/LICENSE">http://www.apache.org/licenses/LICENSE</a>)</li>
<li>Apache Software License 2.0 (available at <a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a>)</li>
<li>IBM Public License 1.0 (available at <a href="http://oss.software.ibm.com/developerworks/opensource/license10.html">http://oss.software.ibm.com/developerworks/opensource/license10.html</a>)</li>
<li>Metro Link Public License 1.00 (available at <a href="http://www.opengroup.org/openmotif/supporters/metrolink/license.html">http://www.opengroup.org/openmotif/supporters/metrolink/license.html</a>)</li>
<li>Mozilla Public License Version 1.1 (available at <a href="http://www.mozilla.org/MPL/MPL-1.1.html">http://www.mozilla.org/MPL/MPL-1.1.html</a>)</li>
</ul>
<p>IT IS YOUR OBLIGATION TO READ AND ACCEPT ALL SUCH TERMS AND
CONDITIONS PRIOR TO USE OF THE CONTENT. If no About, Feature License,
or Feature Update License is provided, please
contact the Eclipse Foundation to determine what terms and conditions
govern that particular Content.</p>
<h3>Cryptography</h3>
<p>Content may contain encryption software. The country in which you
are currently may have restrictions on the import, possession, and use,
and/or re-export to another country, of encryption software. BEFORE
using any encryption software, please check the country's laws,
regulations and policies concerning the import, possession, or use, and
re-export of encryption software, to see if this is permitted.</p>
<small>Java and all Java-based trademarks are trademarks of Sun Microsystems, Inc. in the United States, other countries, or both.</small>
</body></html>

1
solr/licenses/http2-server-9.4.14.v20181114.jar.sha1 Normal file
View File

@ -0,0 +1 @@
a2ce60a90cbf4db91240bb585733e33b1a55110f

202
solr/licenses/http2-server-LICENSE-ASL.txt Normal file
View File

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

111
solr/licenses/http2-server-NOTICE.txt Normal file
View File

@ -0,0 +1,111 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN">
<html><head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>Eclipse.org Software User Agreement</title>
</head><body lang="EN-US" link="blue" vlink="purple">
<h2>Eclipse Foundation Software User Agreement</h2>
<p>March 17, 2005</p>
<h3>Usage Of Content</h3>
<p>THE ECLIPSE FOUNDATION MAKES AVAILABLE SOFTWARE, DOCUMENTATION, INFORMATION AND/OR OTHER MATERIALS FOR OPEN SOURCE PROJECTS
(COLLECTIVELY "CONTENT"). USE OF THE CONTENT IS GOVERNED BY THE TERMS AND CONDITIONS OF THIS AGREEMENT AND/OR THE TERMS AND
CONDITIONS OF LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. BY USING THE CONTENT, YOU AGREE THAT YOUR USE
OF THE CONTENT IS GOVERNED BY THIS AGREEMENT AND/OR THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR
NOTICES INDICATED OR REFERENCED BELOW. IF YOU DO NOT AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT AND THE TERMS AND
CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW, THEN YOU MAY NOT USE THE CONTENT.</p>
<h3>Applicable Licenses</h3>
<p>Unless otherwise indicated, all Content made available by the
Eclipse Foundation is provided to you under the terms and conditions of
the Eclipse Public License Version 1.0 ("EPL"). A copy of the EPL is
provided with this Content and is also available at <a href="http://www.eclipse.org/legal/epl-v10.html">http://www.eclipse.org/legal/epl-v10.html</a>.
For purposes of the EPL, "Program" will mean the Content.</p>
<p>Content includes, but is not limited to, source code, object code,
documentation and other files maintained in the Eclipse.org CVS
repository ("Repository") in CVS modules ("Modules") and made available
as downloadable archives ("Downloads").</p>
<ul>
<li>Content may be structured and packaged into modules to
facilitate delivering, extending, and upgrading the Content. Typical
modules may include plug-ins ("Plug-ins"), plug-in fragments
("Fragments"), and features ("Features").</li>
<li>Each Plug-in or Fragment may be packaged as a sub-directory or JAR (Java&#8482; ARchive) in a directory named "plugins".</li>
<li>A
Feature is a bundle of one or more Plug-ins and/or Fragments and
associated material. Each Feature may be packaged as a sub-directory in
a directory named "features". Within a Feature, files named
"feature.xml" may contain a list of the names and version numbers of
the Plug-ins and/or Fragments associated with that Feature.</li>
<li>Features
may also include other Features ("Included Features"). Within a
Feature, files named "feature.xml" may contain a list of the names and
version numbers of Included Features.</li>
</ul>
<p>The terms and conditions governing Plug-ins and Fragments should be
contained in files named "about.html" ("Abouts"). The terms and
conditions governing Features and
Included Features should be contained in files named "license.html"
("Feature Licenses"). Abouts and Feature Licenses may be located in any
directory of a Download or Module
including, but not limited to the following locations:</p>
<ul>
<li>The top-level (root) directory</li>
<li>Plug-in and Fragment directories</li>
<li>Inside Plug-ins and Fragments packaged as JARs</li>
<li>Sub-directories of the directory named "src" of certain Plug-ins</li>
<li>Feature directories</li>
</ul>
<p>Note: if a Feature made available by the Eclipse Foundation is
installed using the Eclipse Update Manager, you must agree to a license
("Feature Update License") during the
installation process. If the Feature contains Included Features, the
Feature Update License should either provide you with the terms and
conditions governing the Included Features or
inform you where you can locate them. Feature Update Licenses may be
found in the "license" property of files named "feature.properties"
found within a Feature.
Such Abouts, Feature Licenses, and Feature Update Licenses contain the
terms and conditions (or references to such terms and conditions) that
govern your use of the associated Content in
that directory.</p>
<p>THE ABOUTS, FEATURE LICENSES, AND FEATURE UPDATE LICENSES MAY REFER
TO THE EPL OR OTHER LICENSE AGREEMENTS, NOTICES OR TERMS AND
CONDITIONS. SOME OF THESE
OTHER LICENSE AGREEMENTS MAY INCLUDE (BUT ARE NOT LIMITED TO):</p>
<ul>
<li>Common Public License Version 1.0 (available at <a href="http://www.eclipse.org/legal/cpl-v10.html">http://www.eclipse.org/legal/cpl-v10.html</a>)</li>
<li>Apache Software License 1.1 (available at <a href="http://www.apache.org/licenses/LICENSE">http://www.apache.org/licenses/LICENSE</a>)</li>
<li>Apache Software License 2.0 (available at <a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a>)</li>
<li>IBM Public License 1.0 (available at <a href="http://oss.software.ibm.com/developerworks/opensource/license10.html">http://oss.software.ibm.com/developerworks/opensource/license10.html</a>)</li>
<li>Metro Link Public License 1.00 (available at <a href="http://www.opengroup.org/openmotif/supporters/metrolink/license.html">http://www.opengroup.org/openmotif/supporters/metrolink/license.html</a>)</li>
<li>Mozilla Public License Version 1.1 (available at <a href="http://www.mozilla.org/MPL/MPL-1.1.html">http://www.mozilla.org/MPL/MPL-1.1.html</a>)</li>
</ul>
<p>IT IS YOUR OBLIGATION TO READ AND ACCEPT ALL SUCH TERMS AND
CONDITIONS PRIOR TO USE OF THE CONTENT. If no About, Feature License,
or Feature Update License is provided, please
contact the Eclipse Foundation to determine what terms and conditions
govern that particular Content.</p>
<h3>Cryptography</h3>
<p>Content may contain encryption software. The country in which you
are currently may have restrictions on the import, possession, and use,
and/or re-export to another country, of encryption software. BEFORE
using any encryption software, please check the country's laws,
regulations and policies concerning the import, possession, or use, and
re-export of encryption software, to see if this is permitted.</p>
<small>Java and all Java-based trademarks are trademarks of Sun Microsystems, Inc. in the United States, other countries, or both.</small>
</body></html>

1
solr/licenses/jetty-alpn-client-9.4.14.v20181114.jar.sha1 Normal file
View File

@ -0,0 +1 @@
c567eba368e70a0a9aaded14a554a3b25a0a502e

1
solr/licenses/jetty-alpn-java-client-9.4.14.v20181114.jar.sha1 Normal file
View File

@ -0,0 +1 @@
f093d00fc7112bdf471efdd5d909eb9296b3d30d

1
solr/licenses/jetty-alpn-java-server-9.4.14.v20181114.jar.sha1 Normal file
View File

@ -0,0 +1 @@
686cc093a08a2ed2bc2bed059117997c8c760262

1
solr/licenses/jetty-alpn-server-9.4.14.v20181114.jar.sha1 Normal file
View File

@ -0,0 +1 @@
5aa0ca49c6f7cdd4c2c8a628620dc125162213ca

1
solr/licenses/jetty-client-9.4.14.v20181114.jar.sha1 Normal file
View File

@ -0,0 +1 @@
1c46b088e1119928d54ff704fe38fe1b6b6700d0

5
solr/server/etc/jetty-http.xml
View File

@ -31,6 +31,11 @@
<Arg name="config"><Ref refid="httpConfig" /></Arg>
</New>
</Item>
<Item>
<New class="org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory">
<Arg name="config"><Ref refid="httpConfig" /></Arg>
</New>
</Item>
</Array>
</Arg>
<Set name="host"><Property name="jetty.host" /></Set>

26
solr/server/etc/jetty-https.xml
View File

@ -8,6 +8,13 @@
<!-- ============================================================= -->
<Configure id="Server" class="org.eclipse.jetty.server.Server">
<Ref refid="sslContextFactory">
<Set name="CipherComparator">
<Get class="org.eclipse.jetty.http2.HTTP2Cipher" name="COMPARATOR"/>
</Set>
<Set name="useCipherSuitesOrder">true</Set>
</Ref>
<!-- =========================================================== -->
<!-- Add a HTTPS Connector. -->
<!-- Configure an o.e.j.server.ServerConnector with connection -->
@ -29,10 +36,26 @@
<Array type="org.eclipse.jetty.server.ConnectionFactory">
<Item>
<New class="org.eclipse.jetty.server.SslConnectionFactory">
<Arg name="next">http/1.1</Arg>
<Arg name="next">alpn</Arg>
<Arg name="sslContextFactory"><Ref refid="sslContextFactory"/></Arg>
</New>
</Item>
<Item>
<New id="alpn" class="org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory">
<Arg name="protocols">
<Array type="java.lang.String">
<Item>h2</Item>
<Item>http/1.1</Item>
</Array>
</Arg>
<Set name="defaultProtocol">http/1.1</Set>
</New>
</Item>
<Item>
<New class="org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory">
<Arg name="config"><Ref refid="sslHttpConfig"/></Arg>
</New>
</Item>
<Item>
<New class="org.eclipse.jetty.server.HttpConnectionFactory">
<Arg name="config"><Ref refid="sslHttpConfig"/></Arg>
@ -49,4 +72,5 @@
</New>
</Arg>
</Call>
</Configure>

69
solr/server/etc/jetty-https8.xml Normal file
View File

@ -0,0 +1,69 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure_9_0.dtd">
<!-- ============================================================= -->
<!-- Configure a HTTPS connector. -->
<!-- This configuration must be used in conjunction with jetty.xml -->
<!-- and jetty-ssl.xml. -->
<!-- ============================================================= -->
<Configure id="Server" class="org.eclipse.jetty.server.Server">
<!-- =========================================================== -->
<!-- Add a HTTPS Connector. -->
<!-- Configure an o.e.j.server.ServerConnector with connection -->
<!-- factories for TLS (aka SSL) and HTTP to provide HTTPS. -->
<!-- All accepted TLS connections are wired to a HTTP connection.-->
<!-- -->
<!-- Consult the javadoc of o.e.j.server.ServerConnector, -->
<!-- o.e.j.server.SslConnectionFactory and -->
<!-- o.e.j.server.HttpConnectionFactory for all configuration -->
<!-- that may be set here. -->
<!-- =========================================================== -->
<Call id="httpsConnector" name="addConnector">
<Arg>
<New class="org.eclipse.jetty.server.ServerConnector">
<Arg name="server"><Ref refid="Server" /></Arg>
<Arg name="acceptors" type="int"><Property name="solr.jetty.ssl.acceptors" default="-1"/></Arg>
<Arg name="selectors" type="int"><Property name="solr.jetty.ssl.selectors" default="-1"/></Arg>
<Arg name="factories">
<Array type="org.eclipse.jetty.server.ConnectionFactory">
<Item>
<New class="org.eclipse.jetty.server.SslConnectionFactory">
<Arg name="next">http/1.1</Arg>
<Arg name="sslContextFactory"><Ref refid="sslContextFactory"/></Arg>
</New>
</Item>
<Item>
<New class="org.eclipse.jetty.server.HttpConnectionFactory">
<Arg name="config"><Ref refid="sslHttpConfig"/></Arg>
</New>
</Item>
</Array>
</Arg>
<Set name="host"><Property name="solr.jetty.host" /></Set>
<Set name="port"><Property name="solr.jetty.https.port" default="8983" /></Set>
<Set name="idleTimeout"><Property name="solr.jetty.https.timeout" default="120000"/></Set>
<Set name="soLingerTime"><Property name="solr.jetty.https.soLingerTime" default="-1"/></Set>
<Set name="acceptorPriorityDelta"><Property name="solr.jetty.ssl.acceptorPriorityDelta" default="0"/></Set>
<Set name="acceptQueueSize"><Property name="solr.jetty.https.acceptQueueSize" default="0"/></Set>
</New>
</Arg>
</Call>
</Configure>

6
solr/server/ivy.xml
View File

@ -56,6 +56,12 @@
<dependency org="org.eclipse.jetty" name="jetty-util" rev="${/org.eclipse.jetty/jetty-util}" conf="jetty"/>
<dependency org="org.eclipse.jetty" name="jetty-webapp" rev="${/org.eclipse.jetty/jetty-webapp}" conf="jetty"/>
<dependency org="org.eclipse.jetty" name="jetty-xml" rev="${/org.eclipse.jetty/jetty-xml}" conf="jetty"/>
<dependency org="org.eclipse.jetty" name="jetty-alpn-java-server" rev="${/org.eclipse.jetty/jetty-alpn-java-server}" conf="jetty"/>
<dependency org="org.eclipse.jetty" name="jetty-alpn-server" rev="${/org.eclipse.jetty/jetty-alpn-server}" conf="jetty"/>
<dependency org="org.eclipse.jetty.http2" name="http2-server" rev="${/org.eclipse.jetty.http2/http2-server}" conf="jetty"/>
<dependency org="org.eclipse.jetty.http2" name="http2-common" rev="${/org.eclipse.jetty.http2/http2-common}" conf="jetty"/>
<dependency org="org.eclipse.jetty.http2" name="http2-hpack" rev="${/org.eclipse.jetty.http2/http2-hpack}" conf="jetty"/>
<dependency org="javax.servlet" name="javax.servlet-api" rev="${/javax.servlet/javax.servlet-api}" conf="jetty"/>

9
solr/server/modules/https8.mod Normal file
View File

@ -0,0 +1,9 @@
#
# Jetty HTTPS Connector
#
[depend]
ssl
[xml]
etc/jetty-https8.xml

3
solr/solr-ref-guide/src/distributed-requests.adoc
View File

@ -96,9 +96,6 @@ The amount of time in ms that is accepted for binding / connecting a socket. The
`maxConnectionsPerHost`::
The maximum number of concurrent connections that is made to each individual shard in a distributed search. The default is `100000`.
`maxConnections`::
The total maximum number of concurrent connections in distributed searches. The default is `100000`
`corePoolSize`::
The retained lowest limit on the number of threads used in coordinating distributed search. The default is `0`.

3
solr/solr-ref-guide/src/format-of-solr-xml.adoc
View File

@ -174,9 +174,6 @@ The URL scheme to be used in distributed search.
`maxConnectionsPerHost`::
Maximum connections allowed per host. Defaults to `100000`.
`maxConnections`::
Maximum total connections allowed. Defaults to `100000`.
`corePoolSize`::
The initial core size of the threadpool servicing requests. Default is `0`.

37
solr/solr-ref-guide/src/major-changes-in-solr-8.adoc
View File

@ -29,3 +29,40 @@
the old 6.x/7.x scoring. Note that if you have not specified any similarity in schema or use the default
SchemaSimilarityFactory, then LegacyBM25Similarity is automatically selected for 'luceneMatchVersion' < 8.0.0.
See also explanation in Reference Guide chapter "Other Schema Elements".
== Solr 8 Upgrade Planning
Due to the introduction of LIR redesign since Solr 7.3 (SOLR-11702) and the removing of old LIR implementation in Solr 8.
Rolling updates are not possible unless all nodes must be on Solr 7.3 or higher. If not updates can be lost.
Solr nodes can now listen and serve HTTP/2 and HTTP/1 requests. By default, most of internal requests are sent by using HTTP/2
therefore Solr 8.0 nodes can't talk to old nodes (7.x).
However we can follow these steps to do rolling updates:
* Do rolling updates as normally, but the Solr 8.0 nodes must start with `-Dsolr.http1=true` as startup parameter.
By using this parameter internal requests are sent by using HTTP/1.1
* When all nodes are upgraded to 8.0, restart them, this time `-Dsolr.http1` parameter should be removed.
== HTTP/2
Until Solr 8, Solr nodes only support HTTP/1 requests. HTTP/1.1 practically allows only one outstanding request
per TCP connection this means that for sending multiple requests at the same time multiple TCP connections must be
established. This leads to waste of resources on both-sides and long GC-pause. Solr 8 with HTTP/2 support overcomes that problem by allowing
multiple requests can be sent in parallel using a same TCP connection.
However, Java 8 does not include an implementation of ALPN, therefore Solr will start with HTTP/1 only when SSL is enabled.
{solr-javadocs}/solr-solrj/org/apache/solr/client/solrj/impl/Http2SolrClient.html[`Http2SolrClient`]
with HTTP/2 and async capabilities based on Jetty Client is introduced. This client replaced
`HttpSolrClient`] and `ConcurrentUpdateSolrClient` for sending most of internal requests (sent by
`UpdateShardHandler`, `HttpShardHandler`).
However this leads to several changes in configuration and authentication setup
* {solr-javadocs}/solr-core/org/apache/solr/update/UpdateShardHandler.html[`UpdateShardHandler.maxConnections`] parameter is no longer being used
* {solr-javadocs}/solr-core/org/apache/solr/handler/component/HttpShardHandler.html[`HttpShardHandlerFactory.maxConnections`] parameter is no longer being used
* Custom {solr-javadocs}/solr-core/org/apache/solr/security/AuthenticationPlugin.html[`AuthenticationPlugin`] must provide its own setup for
`Http2SolrClient` through implementing
{solr-javadocs}/solr-core/org/apache/solr/security/HttpClientBuilderPlugin.html[`HttpClientBuilderPlugin.setup`],
if not internal requests can't be authenticated

10
solr/solr-ref-guide/src/using-solrj.adoc
View File

@ -86,22 +86,30 @@ Requests are sent in the form of {solr-javadocs}/solr-solrj/org/apache/solr/clie
`SolrClient` has a few concrete implementations, each geared towards a different usage-pattern or resiliency model:
- {solr-javadocs}/solr-solrj/org/apache/solr/client/solrj/impl/HttpSolrClient.html[`HttpSolrClient`] - geared towards query-centric workloads, though also a good general-purpose client. Communicates directly with a single Solr node.
- {solr-javadocs}/solr-solrj/org/apache/solr/client/solrj/impl/Http2SolrClient.html[`Http2SolrClient`] - async, non-blocking and general-purpose client that leverage HTTP/2. This class is experimental therefore its API's might change or be removed in minor versions of SolrJ.
- {solr-javadocs}/solr-solrj/org/apache/solr/client/solrj/impl/LBHttpSolrClient.html[`LBHttpSolrClient`] - balances request load across a list of Solr nodes. Adjusts the list of "in-service" nodes based on node health.
- {solr-javadocs}/solr-solrj/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.html[`LBHttp2SolrClient`] - just like `LBHttpSolrClient` but using `Http2SolrClient` instead. This class is experimental therefore its API's might change or be removed in minor versions of SolrJ.
- {solr-javadocs}/solr-solrj/org/apache/solr/client/solrj/impl/CloudSolrClient.html[`CloudSolrClient`] - geared towards communicating with SolrCloud deployments. Uses already-recorded ZooKeeper state to discover and route requests to healthy Solr nodes.
- {solr-javadocs}/solr-solrj/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.html[`ConcurrentUpdateSolrClient`] - geared towards indexing-centric workloads. Buffers documents internally before sending larger batches to Solr.
- {solr-javadocs}/solr-solrj/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClient.html[`ConcurrentUpdateSolrClient`] - just like `ConcurrentUpdateSolrClient` but using `Http2SolrClient` instead. This class is experimental therefore its API's might change or be removed in minor versions of SolrJ.
=== Common Configuration Options
Most SolrJ configuration happens at the `SolrClient` level. The most common/important of these are discussed below. For comprehensive information on how to tweak your `SolrClient`, see the Javadocs for the involved client, and its corresponding builder object.
==== Base URLs
Most `SolrClient` implementations (with the notable exception of `CloudSolrClient`) require users to specify one or more Solr base URLs, which the client then uses to send HTTP requests to Solr. The path users include on the base URL they provide has an effect on the behavior of the created client from that point on.
Most `SolrClient` implementations (except for `CloudSolrClient` and `Http2SolrClient`) require users to specify one or more Solr base URLs, which the client then uses to send HTTP requests to Solr. The path users include on the base URL they provide has an effect on the behavior of the created client from that point on.
. A URL with a path pointing to a specific core or collection (e.g., `\http://hostname:8983/solr/core1`). When a core or collection is specified in the base URL, subsequent requests made with that client are not required to re-specify the affected collection. However, the client is limited to sending requests to that core/collection, and can not send requests to any others.
. A URL pointing to the root Solr path (e.g., `\http://hostname:8983/solr`). When no core or collection is specified in the base URL, requests can be made to any core/collection, but the affected core/collection must be specified on all requests.
Generally speaking, if your `SolrClient` will only be used on a single core/collection, including that entity in the path is the most convenient. Where more flexibility is required, the collection/core should be excluded.
==== Base URLs of Http2SolrClient
Since `Http2SolrClient` can manages connections to different nodes efficiently. `Http2SolrClient`
does not require a `baseUrl`. In case of `baseUrl` is not provided, `SolrRequest.basePath` must be set, so
`Http2SolrClient` can know which node it will request to. If not an `IllegalArgumentException` will be thrown.
==== Timeouts
All `SolrClient` implementations allow users to specify the connection and read timeouts for communicating with Solr. These are provided at client creation time, as in the example below:

12
solr/solrj/ivy.xml
View File

@ -40,6 +40,18 @@
<dependency org="org.slf4j" name="slf4j-api" rev="${/org.slf4j/slf4j-api}" conf="compile"/>
<dependency org="org.slf4j" name="jcl-over-slf4j" rev="${/org.slf4j/jcl-over-slf4j}" conf="compile"/>
<dependency org="org.eclipse.jetty.http2" name="http2-client" rev="${/org.eclipse.jetty.http2/http2-client}" conf="compile"/>
<dependency org="org.eclipse.jetty.http2" name="http2-http-client-transport" rev="${/org.eclipse.jetty.http2/http2-http-client-transport}" conf="compile"/>
<dependency org="org.eclipse.jetty.http2" name="http2-common" rev="${/org.eclipse.jetty.http2/http2-common}" conf="compile"/>
<dependency org="org.eclipse.jetty.http2" name="http2-hpack" rev="${/org.eclipse.jetty.http2/http2-hpack}" conf="compile"/>
<dependency org="org.eclipse.jetty" name="jetty-client" rev="${/org.eclipse.jetty/jetty-client}" conf="compile"/>
<dependency org="org.eclipse.jetty" name="jetty-util" rev="${/org.eclipse.jetty/jetty-util}" conf="compile"/>
<dependency org="org.eclipse.jetty" name="jetty-http" rev="${/org.eclipse.jetty/jetty-http}" conf="compile"/>
<dependency org="org.eclipse.jetty" name="jetty-io" rev="${/org.eclipse.jetty/jetty-io}" conf="compile"/>
<dependency org="org.eclipse.jetty" name="jetty-alpn-java-client" rev="${/org.eclipse.jetty/jetty-alpn-java-client}" conf="compile"/>
<dependency org="org.eclipse.jetty" name="jetty-alpn-client" rev="${/org.eclipse.jetty/jetty-alpn-client}" conf="compile"/>
<dependency org="org.apache.logging.log4j" name="log4j-slf4j-impl" rev="${/org.apache.logging.log4j/log4j-slf4j-impl}" conf="test"/>
<dependency org="org.mockito" name="mockito-core" rev="${/org.mockito/mockito-core}" conf="test"/>

11
solr/solrj/src/java/org/apache/solr/client/solrj/SolrRequest.java
View File

@ -88,6 +88,8 @@ public abstract class SolrRequest<T extends SolrResponse> implements Serializabl
private String basicAuthUser, basicAuthPwd;
private String basePath;
public SolrRequest setBasicAuthCredentials(String user, String password) {
this.basicAuthUser = user;
this.basicAuthPwd = password;
@ -226,4 +228,13 @@ public abstract class SolrRequest<T extends SolrResponse> implements Serializabl
return getParams() == null ? null : getParams().get("collection");
}
public void setBasePath(String path) {
if (path.endsWith("/")) path = path.substring(0, path.length() - 1);
this.basePath = path;
}
public String getBasePath() {
return basePath;
}
}

11
solr/core/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java → solr/solrj/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
View File

@ -16,11 +16,10 @@
*/
package org.apache.solr.client.solrj.embedded;
import org.apache.lucene.util.Constants;
import org.eclipse.jetty.util.ssl.SslContextFactory;
/**
* Encapsulates settings related to SSL Configuration for an embedded Jetty Server.
* Encapsulates settings related to SSL Configuration.
* NOTE: all other settings are ignored if {@link #isSSLMode} is false.
* @see #setUseSSL
*/
@ -35,13 +34,7 @@ public class SSLConfig {
/** NOTE: all other settings are ignored if useSSL is false; trustStore settings are ignored if clientAuth is false */
public SSLConfig(boolean useSSL, boolean clientAuth, String keyStore, String keyStorePassword, String trustStore, String trustStorePassword) {
// @AwaitsFix: SOLR-12988 - ssl issues on Java 11/12
if (Constants.JRE_IS_MINIMUM_JAVA11) {
this.useSsl = false;
} else {
this.useSsl = useSSL;
}
this.useSsl = useSSL;
this.clientAuth = clientAuth;
this.keyStore = keyStore;
this.keyStorePassword = keyStorePassword;

24
solr/solrj/src/java/org/apache/solr/client/solrj/embedded/package-info.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* SolrJ client implementations for embedded solr access.
*/
package org.apache.solr.client.solrj.embedded;

690
solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClient.java Normal file
View File

@ -0,0 +1,690 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Queue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.Update;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.eclipse.jetty.client.api.Response;
import org.eclipse.jetty.client.util.InputStreamResponseListener;
import org.eclipse.jetty.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
/**
* @lucene.experimental
*/
public class ConcurrentUpdateHttp2SolrClient extends SolrClient {
private static final long serialVersionUID = 1L;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final Update END_UPDATE = new Update(null, null);
private Http2SolrClient client;
private final String basePath;
private final CustomBlockingQueue<Update> queue;
private final ExecutorService scheduler;
private final Queue<Runner> runners;
private final int threadCount;
private boolean shutdownClient;
private boolean shutdownExecutor;
private int pollQueueTime = 250;
private final boolean streamDeletes;
private volatile boolean closed;
private volatile CountDownLatch lock = null; // used to block everything
private static class CustomBlockingQueue<E> implements Iterable<E>{
private final BlockingQueue<E> queue;
private final Semaphore available;
private final int queueSize;
private final E backdoorE;
public CustomBlockingQueue(int queueSize, int maxConsumers, E backdoorE) {
queue = new LinkedBlockingQueue<>();
available = new Semaphore(queueSize);
this.queueSize = queueSize;
this.backdoorE = backdoorE;
}
public boolean offer(E e) {
boolean success = available.tryAcquire();
if (success) {
queue.offer(e);
}
return success;
}
public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException {
boolean success = available.tryAcquire(timeout, unit);
if (success) {
queue.offer(e);
}
return success;
}
public boolean isEmpty() {
return size() == 0;
}
public E poll(int timeout, TimeUnit unit) throws InterruptedException {
E e = queue.poll(timeout, unit);
if (e == null) {
return null;
}
if (e == backdoorE)
return null;
available.release();
return e;
}
public boolean add(E e) {
boolean success = available.tryAcquire();
if (success) {
queue.add(e);
} else {
throw new IllegalStateException("Queue is full");
}
return true;
}
public int size() {
return queueSize - available.availablePermits();
}
public int remainingCapacity() {
return available.availablePermits();
}
@Override
public Iterator<E> iterator() {
return queue.iterator();
}
public void backdoorOffer() {
queue.offer(backdoorE);
}
}
protected ConcurrentUpdateHttp2SolrClient(Builder builder) {
this.client = builder.client;
this.shutdownClient = builder.closeHttp2Client;
this.threadCount = builder.threadCount;
this.queue = new CustomBlockingQueue<>(builder.queueSize, threadCount, END_UPDATE);
this.runners = new LinkedList<>();
this.streamDeletes = builder.streamDeletes;
this.basePath = builder.baseSolrUrl;
if (builder.executorService != null) {
this.scheduler = builder.executorService;
this.shutdownExecutor = false;
} else {
this.scheduler = ExecutorUtil.newMDCAwareCachedThreadPool(new SolrjNamedThreadFactory("concurrentUpdateScheduler"));
this.shutdownExecutor = true;
}
}
/**
* Opens a connection and sends everything...
*/
class Runner implements Runnable {
@Override
public void run() {
log.debug("starting runner: {}", this);
// This loop is so we can continue if an element was added to the queue after the last runner exited.
for (;;) {
try {
sendUpdateStream();
} catch (Throwable e) {
if (e instanceof OutOfMemoryError) {
throw (OutOfMemoryError) e;
}
handleError(e);
} finally {
synchronized (runners) {
// check to see if anything else was added to the queue
if (runners.size() == 1 && !queue.isEmpty() && !scheduler.isShutdown()) {
// If there is something else to process, keep last runner alive by staying in the loop.
} else {
runners.remove(this);
if (runners.isEmpty()) {
// notify anyone waiting in blockUntilFinished
runners.notifyAll();
}
break;
}
}
}
}
log.debug("finished: {}", this);
}
//
// Pull from the queue multiple times and streams over a single connection.
// Exits on exception, interruption, or an empty queue to pull from.
//
void sendUpdateStream() throws Exception {
try {
while (!queue.isEmpty()) {
InputStream rspBody = null;
try {
Update update;
notifyQueueAndRunnersIfEmptyQueue();
update = queue.poll(pollQueueTime, TimeUnit.MILLISECONDS);
if (update == null) {
break;
}
InputStreamResponseListener responseListener = null;
try (Http2SolrClient.OutStream out = client.initOutStream(basePath, update.getRequest(),
update.getCollection())) {
Update upd = update;
while (upd != null) {
UpdateRequest req = upd.getRequest();
if (!out.belongToThisStream(req, upd.getCollection())) {
queue.add(upd); // Request has different params or destination core/collection, return to queue
break;
}
client.send(out, upd.getRequest(), upd.getCollection());
out.flush();
notifyQueueAndRunnersIfEmptyQueue();
upd = queue.poll(pollQueueTime, TimeUnit.MILLISECONDS);
}
responseListener = out.getResponseListener();
}
Response response = responseListener.get(client.getIdleTimeout(), TimeUnit.MILLISECONDS);
rspBody = responseListener.getInputStream();
int statusCode = response.getStatus();
if (statusCode != HttpStatus.OK_200) {
StringBuilder msg = new StringBuilder();
msg.append(response.getReason());
msg.append("\n\n\n\n");
msg.append("request: ").append(basePath);
SolrException solrExc;
NamedList<String> metadata = null;
// parse out the metadata from the SolrException
try {
String encoding = "UTF-8"; // default
NamedList<Object> resp = client.getParser().processResponse(rspBody, encoding);
NamedList<Object> error = (NamedList<Object>) resp.get("error");
if (error != null) {
metadata = (NamedList<String>) error.get("metadata");
String remoteMsg = (String) error.get("msg");
if (remoteMsg != null) {
msg.append("\nRemote error message: ");
msg.append(remoteMsg);
}
}
} catch (Exception exc) {
// don't want to fail to report error if parsing the response fails
log.warn("Failed to parse error response from " + basePath + " due to: " + exc);
} finally {
solrExc = new HttpSolrClient.RemoteSolrException(basePath , statusCode, msg.toString(), null);
if (metadata != null) {
solrExc.setMetadata(metadata);
}
}
handleError(solrExc);
} else {
onSuccess(response, rspBody);
}
} finally {
try {
if (rspBody != null) {
while (rspBody.read() != -1) {}
}
} catch (Exception e) {
log.error("Error consuming and closing http response stream.", e);
}
notifyQueueAndRunnersIfEmptyQueue();
}
}
} catch (InterruptedException e) {
log.error("Interrupted on polling from queue", e);
}
}
}
private void notifyQueueAndRunnersIfEmptyQueue() {
if (queue.size() == 0) {
synchronized (queue) {
// queue may be empty
queue.notifyAll();
}
synchronized (runners) {
// we notify runners too - if there is a high queue poll time and this is the update
// that emptied the queue, we make an attempt to avoid the 250ms timeout in blockUntilFinished
runners.notifyAll();
}
}
}
// *must* be called with runners monitor held, e.g. synchronized(runners){ addRunner() }
private void addRunner() {
MDC.put("ConcurrentUpdateHttp2SolrClient.url", client.getBaseURL());
try {
Runner r = new Runner();
runners.add(r);
try {
scheduler.execute(r); // this can throw an exception if the scheduler has been shutdown, but that should be fine.
} catch (RuntimeException e) {
runners.remove(r);
throw e;
}
} finally {
MDC.remove("ConcurrentUpdateHttp2SolrClient.url");
}
}
@Override
public NamedList<Object> request(final SolrRequest request, String collection)
throws SolrServerException, IOException {
if (!(request instanceof UpdateRequest)) {
request.setBasePath(basePath);
return client.request(request, collection);
}
UpdateRequest req = (UpdateRequest) request;
req.setBasePath(basePath);
// this happens for commit...
if (streamDeletes) {
if ((req.getDocuments() == null || req.getDocuments().isEmpty())
&& (req.getDeleteById() == null || req.getDeleteById().isEmpty())
&& (req.getDeleteByIdMap() == null || req.getDeleteByIdMap().isEmpty())) {
if (req.getDeleteQuery() == null) {
blockUntilFinished();
return client.request(request, collection);
}
}
} else {
if ((req.getDocuments() == null || req.getDocuments().isEmpty())) {
blockUntilFinished();
return client.request(request, collection);
}
}
SolrParams params = req.getParams();
if (params != null) {
// check if it is waiting for the searcher
if (params.getBool(UpdateParams.WAIT_SEARCHER, false)) {
log.info("blocking for commit/optimize");
blockUntilFinished(); // empty the queue
return client.request(request, collection);
}
}
try {
CountDownLatch tmpLock = lock;
if (tmpLock != null) {
tmpLock.await();
}
Update update = new Update(req, collection);
boolean success = queue.offer(update);
for (;;) {
synchronized (runners) {
// see if queue is half full and we can add more runners
// special case: if only using a threadCount of 1 and the queue
// is filling up, allow 1 add'l runner to help process the queue
if (runners.isEmpty() || (queue.remainingCapacity() < queue.size() && runners.size() < threadCount))
{
// We need more runners, so start a new one.
addRunner();
} else {
// break out of the retry loop if we added the element to the queue
// successfully, *and*
// while we are still holding the runners lock to prevent race
// conditions.
if (success)
break;
}
}
// Retry to add to the queue w/o the runners lock held (else we risk
// temporary deadlock)
// This retry could also fail because
// 1) existing runners were not able to take off any new elements in the
// queue
// 2) the queue was filled back up since our last try
// If we succeed, the queue may have been completely emptied, and all
// runners stopped.
// In all cases, we should loop back to the top to see if we need to
// start more runners.
//
if (!success) {
success = queue.offer(update, 100, TimeUnit.MILLISECONDS);
}
}
} catch (InterruptedException e) {
log.error("interrupted", e);
throw new IOException(e.getLocalizedMessage());
}
// RETURN A DUMMY result
NamedList<Object> dummy = new NamedList<>();
dummy.add("NOTE", "the request is processed in a background stream");
return dummy;
}
public synchronized void blockUntilFinished() {
lock = new CountDownLatch(1);
try {
waitForEmptyQueue();
interruptRunnerThreadsPolling();
synchronized (runners) {
// NOTE: if the executor is shut down, runners may never become empty (a scheduled task may never be run,
// which means it would never remove itself from the runners list. This is why we don't wait forever
// and periodically check if the scheduler is shutting down.
int loopCount = 0;
while (!runners.isEmpty()) {
if (scheduler.isShutdown())
break;
loopCount++;
// Need to check if the queue is empty before really considering this is finished (SOLR-4260)
int queueSize = queue.size();
if (queueSize > 0 && runners.isEmpty()) {
// TODO: can this still happen?
log.warn("No more runners, but queue still has " +
queueSize + " adding more runners to process remaining requests on queue");
addRunner();
}
interruptRunnerThreadsPolling();
// try to avoid the worst case wait timeout
// without bad spin
int timeout;
if (loopCount < 3) {
timeout = 10;
} else if (loopCount < 10) {
timeout = 25;
} else {
timeout = 250;
}
try {
runners.wait(timeout);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
} finally {
lock.countDown();
lock = null;
}
}
private void waitForEmptyQueue() {
boolean threadInterrupted = Thread.currentThread().isInterrupted();
while (!queue.isEmpty()) {
if (scheduler.isTerminated()) {
log.warn("The task queue still has elements but the update scheduler {} is terminated. Can't process any more tasks. "
+ "Queue size: {}, Runners: {}. Current thread Interrupted? {}", scheduler, queue.size(), runners.size(), threadInterrupted);
break;
}
synchronized (runners) {
int queueSize = queue.size();
if (queueSize > 0 && runners.isEmpty()) {
log.warn("No more runners, but queue still has " +
queueSize + " adding more runners to process remaining requests on queue");
addRunner();
}
}
synchronized (queue) {
try {
queue.wait(250);
} catch (InterruptedException e) {
// If we set the thread as interrupted again, the next time the wait it's called i t's going to return immediately
threadInterrupted = true;
log.warn("Thread interrupted while waiting for update queue to be empty. There are still {} elements in the queue.",
queue.size());
}
}
}
if (threadInterrupted) {
Thread.currentThread().interrupt();
}
}
public void handleError(Throwable ex) {
log.error("error", ex);
}
/**
* Intended to be used as an extension point for doing post processing after a request completes.
*/
public void onSuccess(Response resp, InputStream respBody) {
// no-op by design, override to add functionality
}
@Override
public synchronized void close() {
if (closed) {
interruptRunnerThreadsPolling();
return;
}
closed = true;
try {
if (shutdownExecutor) {
scheduler.shutdown();
interruptRunnerThreadsPolling();
try {
if (!scheduler.awaitTermination(60, TimeUnit.SECONDS)) {
scheduler.shutdownNow();
if (!scheduler.awaitTermination(60, TimeUnit.SECONDS)) log
.error("ExecutorService did not terminate");
}
} catch (InterruptedException ie) {
scheduler.shutdownNow();
Thread.currentThread().interrupt();
}
} else {
interruptRunnerThreadsPolling();
}
} finally {
if (shutdownClient)
client.close();
}
}
private void interruptRunnerThreadsPolling() {
synchronized (runners) {
for (Runner ignored : runners) {
queue.backdoorOffer();
}
}
}
public void shutdownNow() {
if (closed) {
return;
}
closed = true;
if (shutdownExecutor) {
scheduler.shutdown();
interruptRunnerThreadsPolling();
scheduler.shutdownNow(); // Cancel currently executing tasks
try {
if (!scheduler.awaitTermination(30, TimeUnit.SECONDS))
log.error("ExecutorService did not terminate");
} catch (InterruptedException ie) {
scheduler.shutdownNow();
Thread.currentThread().interrupt();
}
} else {
interruptRunnerThreadsPolling();
}
}
/**
* @param pollQueueTime time for an open connection to wait for updates when
* the queue is empty.
*/
public void setPollQueueTime(int pollQueueTime) {
this.pollQueueTime = pollQueueTime;
}
/**
* Constructs {@link ConcurrentUpdateHttp2SolrClient} instances from provided configuration.
*/
public static class Builder {
protected Http2SolrClient client;
protected String baseSolrUrl;
protected int queueSize = 10;
protected int threadCount;
protected ExecutorService executorService;
protected boolean streamDeletes;
protected boolean closeHttp2Client;
public Builder(String baseSolrUrl, Http2SolrClient client) {
this(baseSolrUrl, client, false);
}
public Builder(String baseSolrUrl, Http2SolrClient client, boolean closeHttp2Client) {
this.baseSolrUrl = baseSolrUrl;
this.client = client;
this.closeHttp2Client = closeHttp2Client;
}
/**
* The maximum number of requests buffered by the SolrClient's internal queue before being processed by background threads.
*
* This value should be carefully paired with the number of queue-consumer threads. A queue with a maximum size
* set too high may require more memory. A queue with a maximum size set too low may suffer decreased throughput
* as {@link ConcurrentUpdateHttp2SolrClient#request(SolrRequest)} calls block waiting to add requests to the queue.
*
* If not set, this defaults to 10.
*
* @see #withThreadCount(int)
*/
public Builder withQueueSize(int queueSize) {
if (queueSize <= 0) {
throw new IllegalArgumentException("queueSize must be a positive integer.");
}
this.queueSize = queueSize;
return this;
}
/**
* The maximum number of threads used to empty {@link ConcurrentUpdateHttp2SolrClient}s queue.
*
* Threads are created when documents are added to the client's internal queue and exit when no updates remain in
* the queue.
* <p>
* This value should be carefully paired with the maximum queue capacity. A client with too few threads may suffer
* decreased throughput as the queue fills up and {@link ConcurrentUpdateHttp2SolrClient#request(SolrRequest)} calls
* block waiting to add requests to the queue.
*/
public Builder withThreadCount(int threadCount) {
if (threadCount <= 0) {
throw new IllegalArgumentException("threadCount must be a positive integer.");
}
this.threadCount = threadCount;
return this;
}
/**
* Provides the {@link ExecutorService} for the created client to use when servicing the update-request queue.
*/
public Builder withExecutorService(ExecutorService executorService) {
this.executorService = executorService;
return this;
}
/**
* Configures created clients to always stream delete requests.
*
* Streamed deletes are put into the update-queue and executed like any other update request.
*/
public Builder alwaysStreamDeletes() {
this.streamDeletes = true;
return this;
}
/**
* Configures created clients to not stream delete requests.
*
* With this option set when the created ConcurrentUpdateHttp2SolrClient sents a delete request it will first will lock
* the queue and block until all queued updates have been sent, and then send the delete request.
*/
public Builder neverStreamDeletes() {
this.streamDeletes = false;
return this;
}
/**
* Create a {@link ConcurrentUpdateHttp2SolrClient} based on the provided configuration options.
*/
public ConcurrentUpdateHttp2SolrClient build() {
if (baseSolrUrl == null) {
throw new IllegalArgumentException("Cannot create HttpSolrClient without a valid baseSolrUrl!");
}
return new ConcurrentUpdateHttp2SolrClient(this);
}
}
}

979
solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java Normal file
View File

@ -0,0 +1,979 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.lang.invoke.MethodHandles;
import java.net.ConnectException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Phaser;
import java.util.concurrent.Semaphore;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpStatus;
import org.apache.http.entity.ContentType;
import org.apache.solr.client.solrj.ResponseParser;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.V2RequestSupport;
import org.apache.solr.client.solrj.embedded.SSLConfig;
import org.apache.solr.client.solrj.request.RequestWriter;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.request.V2Request;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.client.solrj.util.Constants;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.StringUtils;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.Base64;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.eclipse.jetty.client.HttpClient;
import org.eclipse.jetty.client.HttpClientTransport;
import org.eclipse.jetty.client.ProtocolHandlers;
import org.eclipse.jetty.client.api.Request;
import org.eclipse.jetty.client.api.Response;
import org.eclipse.jetty.client.api.Result;
import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
import org.eclipse.jetty.client.util.BufferingResponseListener;
import org.eclipse.jetty.client.util.BytesContentProvider;
import org.eclipse.jetty.client.util.FormContentProvider;
import org.eclipse.jetty.client.util.InputStreamContentProvider;
import org.eclipse.jetty.client.util.InputStreamResponseListener;
import org.eclipse.jetty.client.util.MultiPartContentProvider;
import org.eclipse.jetty.client.util.OutputStreamContentProvider;
import org.eclipse.jetty.client.util.StringContentProvider;
import org.eclipse.jetty.http.HttpField;
import org.eclipse.jetty.http.HttpFields;
import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http2.client.HTTP2Client;
import org.eclipse.jetty.http2.client.http.HttpClientTransportOverHTTP2;
import org.eclipse.jetty.util.BlockingArrayQueue;
import org.eclipse.jetty.util.Fields;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.common.util.Utils.getObjectByPath;
// TODO: error handling, small Http2SolrClient features, security, ssl
/**
* @lucene.experimental
*/
public class Http2SolrClient extends SolrClient {
public static final String REQ_PRINCIPAL_KEY = "solr-req-principal";
private static volatile SSLConfig defaultSSLConfig;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String AGENT = "Solr[" + Http2SolrClient.class.getName() + "] 2.0";
private static final String UTF_8 = StandardCharsets.UTF_8.name();
private static final String DEFAULT_PATH = "/select";
private static final List<String> errPath = Arrays.asList("metadata", "error-class");
private HttpClient httpClient;
private volatile Set<String> queryParams = Collections.emptySet();
private int idleTimeout;
private ResponseParser parser = new BinaryResponseParser();
private volatile RequestWriter requestWriter = new BinaryRequestWriter();
private List<HttpListenerFactory> listenerFactory = new LinkedList<>();
private AsyncTracker asyncTracker = new AsyncTracker();
/**
* The URL of the Solr server.
*/
private String serverBaseUrl;
private boolean closeClient;
protected Http2SolrClient(String serverBaseUrl, Builder builder) {
if (serverBaseUrl != null) {
if (!serverBaseUrl.equals("/") && serverBaseUrl.endsWith("/")) {
serverBaseUrl = serverBaseUrl.substring(0, serverBaseUrl.length() - 1);
}
if (serverBaseUrl.startsWith("//")) {
serverBaseUrl = serverBaseUrl.substring(1, serverBaseUrl.length());
}
this.serverBaseUrl = serverBaseUrl;
}
if (builder.idleTimeout != null) idleTimeout = builder.idleTimeout;
else idleTimeout = HttpClientUtil.DEFAULT_SO_TIMEOUT;
if (builder.httpClient == null) {
httpClient = createHttpClient(builder);
closeClient = true;
} else {
httpClient = builder.httpClient;
}
if (!httpClient.isStarted()) {
try {
httpClient.start();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
assert ObjectReleaseTracker.track(this);
}
public void addListenerFactory(HttpListenerFactory factory) {
this.listenerFactory.add(factory);
}
HttpClient getHttpClient() {
return httpClient;
}
ProtocolHandlers getProtocolHandlers() {
return httpClient.getProtocolHandlers();
}
private HttpClient createHttpClient(Builder builder) {
HttpClient httpClient;
BlockingArrayQueue<Runnable> queue = new BlockingArrayQueue<>(256, 256);
ThreadPoolExecutor httpClientExecutor = new ExecutorUtil.MDCAwareThreadPoolExecutor(32,
256, 60, TimeUnit.SECONDS, queue, new SolrjNamedThreadFactory("h2sc"));
SslContextFactory sslContextFactory;
boolean ssl;
if (builder.sslConfig == null) {
sslContextFactory = getDefaultSslContextFactory();
ssl = sslContextFactory.getTrustStore() != null || sslContextFactory.getTrustStorePath() != null;
} else {
sslContextFactory = builder.sslConfig.createContextFactory();
ssl = true;
}
boolean sslOnJava8OrLower = ssl && !Constants.JRE_IS_MINIMUM_JAVA9;
HttpClientTransport transport;
if (builder.useHttp1_1 || sslOnJava8OrLower) {
if (sslOnJava8OrLower && !builder.useHttp1_1) {
log.warn("Create Http2SolrClient with HTTP/1.1 transport since Java 8 or lower versions does not support SSL + HTTP/2");
} else {
log.debug("Create Http2SolrClient with HTTP/1.1 transport");
}
transport = new HttpClientTransportOverHTTP(2);
httpClient = new HttpClient(transport, sslContextFactory);
if (builder.maxConnectionsPerHost != null) httpClient.setMaxConnectionsPerDestination(builder.maxConnectionsPerHost);
} else {
log.debug("Create Http2SolrClient with HTTP/2 transport");
HTTP2Client http2client = new HTTP2Client();
transport = new HttpClientTransportOverHTTP2(http2client);
httpClient = new HttpClient(transport, sslContextFactory);
httpClient.setMaxConnectionsPerDestination(4);
}
httpClient.setExecutor(httpClientExecutor);
httpClient.setStrictEventOrdering(false);
httpClient.setConnectBlocking(true);
httpClient.setFollowRedirects(false);
httpClient.setMaxRequestsQueuedPerDestination(asyncTracker.getMaxRequestsQueuedPerDestination());
httpClient.setUserAgentField(new HttpField(HttpHeader.USER_AGENT, AGENT));
if (builder.idleTimeout != null) httpClient.setIdleTimeout(builder.idleTimeout);
if (builder.connectionTimeout != null) httpClient.setConnectTimeout(builder.connectionTimeout);
return httpClient;
}
public void close() {
// we wait for async requests, so far devs don't want to give sugar for this
asyncTracker.waitForComplete();
if (closeClient) {
try {
ExecutorService executor = (ExecutorService) httpClient.getExecutor();
httpClient.setStopTimeout(1000);
httpClient.stop();
ExecutorUtil.shutdownAndAwaitTermination(executor);
} catch (Exception e) {
throw new RuntimeException("Exception on closing client", e);
}
}
assert ObjectReleaseTracker.release(this);
System.out.println("Done close " + httpClient.getExecutor().toString());
}
public boolean isV2ApiRequest(final SolrRequest request) {
return request instanceof V2Request || request.getPath().contains("/____v2");
}
public long getIdleTimeout() {
return idleTimeout;
}
public static class OutStream implements Closeable{
private final String origCollection;
private final ModifiableSolrParams origParams;
private final OutputStreamContentProvider outProvider;
private final InputStreamResponseListener responseListener;
private final boolean isXml;
public OutStream(String origCollection, ModifiableSolrParams origParams,
OutputStreamContentProvider outProvider, InputStreamResponseListener responseListener, boolean isXml) {
this.origCollection = origCollection;
this.origParams = origParams;
this.outProvider = outProvider;
this.responseListener = responseListener;
this.isXml = isXml;
}
boolean belongToThisStream(SolrRequest solrRequest, String collection) {
ModifiableSolrParams solrParams = new ModifiableSolrParams(solrRequest.getParams());
if (!origParams.toNamedList().equals(solrParams.toNamedList()) || !StringUtils.equals(origCollection, collection)) {
return false;
}
return true;
}
public void write(byte b[]) throws IOException {
this.outProvider.getOutputStream().write(b);
}
public void flush() throws IOException {
this.outProvider.getOutputStream().flush();
}
@Override
public void close() throws IOException {
if (isXml) {
write("</stream>".getBytes(StandardCharsets.UTF_8));
}
this.outProvider.getOutputStream().close();
}
//TODO this class should be hidden
public InputStreamResponseListener getResponseListener() {
return responseListener;
}
}
public OutStream initOutStream(String baseUrl,
UpdateRequest updateRequest,
String collection) throws IOException {
String contentType = requestWriter.getUpdateContentType();
final ModifiableSolrParams origParams = new ModifiableSolrParams(updateRequest.getParams());
// The parser 'wt=' and 'version=' params are used instead of the
// original params
ModifiableSolrParams requestParams = new ModifiableSolrParams(origParams);
requestParams.set(CommonParams.WT, parser.getWriterType());
requestParams.set(CommonParams.VERSION, parser.getVersion());
String basePath = baseUrl;
if (collection != null)
basePath += "/" + collection;
if (!basePath.endsWith("/"))
basePath += "/";
OutputStreamContentProvider provider = new OutputStreamContentProvider();
Request postRequest = httpClient
.newRequest(basePath + "update"
+ requestParams.toQueryString())
.method(HttpMethod.POST)
.header("User-Agent", HttpSolrClient.AGENT)
.header("Content-Type", contentType)
.content(provider);
setListeners(updateRequest, postRequest);
InputStreamResponseListener responseListener = new InputStreamResponseListener();
postRequest.send(responseListener);
boolean isXml = ClientUtils.TEXT_XML.equals(requestWriter.getUpdateContentType());
OutStream outStream = new OutStream(collection, origParams, provider, responseListener,
isXml);
if (isXml) {
outStream.write("<stream>".getBytes(StandardCharsets.UTF_8));
}
return outStream;
}
public void send(OutStream outStream, SolrRequest req, String collection) throws IOException {
assert outStream.belongToThisStream(req, collection);
this.requestWriter.write(req, outStream.outProvider.getOutputStream());
if (outStream.isXml) {
// check for commit or optimize
SolrParams params = req.getParams();
if (params != null) {
String fmt = null;
if (params.getBool(UpdateParams.OPTIMIZE, false)) {
fmt = "<optimize waitSearcher=\"%s\" />";
} else if (params.getBool(UpdateParams.COMMIT, false)) {
fmt = "<commit waitSearcher=\"%s\" />";
}
if (fmt != null) {
byte[] content = String.format(Locale.ROOT,
fmt, params.getBool(UpdateParams.WAIT_SEARCHER, false)
+ "")
.getBytes(StandardCharsets.UTF_8);
outStream.write(content);
}
}
}
outStream.flush();
}
public NamedList<Object> request(SolrRequest solrRequest,
String collection,
OnComplete onComplete) throws IOException, SolrServerException {
Request req = makeRequest(solrRequest, collection);
final ResponseParser parser = solrRequest.getResponseParser() == null
? this.parser: solrRequest.getResponseParser();
if (onComplete != null) {
// This async call only suitable for indexing since the response size is limited by 5MB
req.onRequestQueued(asyncTracker.queuedListener)
.onComplete(asyncTracker.completeListener).send(new BufferingResponseListener(5 * 1024 * 1024) {
@Override
public void onComplete(Result result) {
if (result.isFailed()) {
onComplete.onFailure(result.getFailure());
return;
}
NamedList<Object> rsp;
try {
InputStream is = getContentAsInputStream();
assert ObjectReleaseTracker.track(is);
rsp = processErrorsAndResponse(result.getResponse(),
parser, is, getEncoding(), isV2ApiRequest(solrRequest));
onComplete.onSuccess(rsp);
} catch (Exception e) {
onComplete.onFailure(e);
}
}
});
return null;
} else {
try {
InputStreamResponseListener listener = new InputStreamResponseListener();
req.send(listener);
Response response = listener.get(idleTimeout, TimeUnit.MILLISECONDS);
InputStream is = listener.getInputStream();
assert ObjectReleaseTracker.track(is);
return processErrorsAndResponse(response, parser, is, getEncoding(response), isV2ApiRequest(solrRequest));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException(e);
} catch (TimeoutException e) {
throw new SolrServerException(
"Timeout occured while waiting response from server at: " + req.getURI(), e);
} catch (ExecutionException e) {
Throwable cause = e.getCause();
if (cause instanceof ConnectException) {
throw new SolrServerException("Server refused connection at: " + req.getURI(), cause);
}
if (cause instanceof SolrServerException) {
throw (SolrServerException) cause;
} else if (cause instanceof IOException) {
throw new SolrServerException(
"IOException occured when talking to server at: " + getBaseURL(), cause);
}
throw new SolrServerException(cause.getMessage(), cause);
}
}
}
private String getEncoding(Response response) {
String contentType = response.getHeaders().get(HttpHeader.CONTENT_TYPE);
if (contentType != null) {
String charset = "charset=";
int index = contentType.toLowerCase(Locale.ENGLISH).indexOf(charset);
if (index > 0) {
String encoding = contentType.substring(index + charset.length());
// Sometimes charsets arrive with an ending semicolon.
int semicolon = encoding.indexOf(';');
if (semicolon > 0)
encoding = encoding.substring(0, semicolon).trim();
// Sometimes charsets are quoted.
int lastIndex = encoding.length() - 1;
if (encoding.charAt(0) == '"' && encoding.charAt(lastIndex) == '"')
encoding = encoding.substring(1, lastIndex).trim();
return encoding;
}
}
return null;
}
private void setBasicAuthHeader(SolrRequest solrRequest, Request req) {
if (solrRequest.getBasicAuthUser() != null && solrRequest.getBasicAuthPassword() != null) {
String userPass = solrRequest.getBasicAuthUser() + ":" + solrRequest.getBasicAuthPassword();
String encoded = Base64.byteArrayToBase64(userPass.getBytes(StandardCharsets.UTF_8));
req.header("Authorization", "Basic " + encoded);
}
}
private Request makeRequest(SolrRequest solrRequest, String collection)
throws SolrServerException, IOException {
Request req = createRequest(solrRequest, collection);
setListeners(solrRequest, req);
if (solrRequest.getUserPrincipal() != null) {
req.attribute(REQ_PRINCIPAL_KEY, solrRequest.getUserPrincipal());
}
return req;
}
private void setListeners(SolrRequest solrRequest, Request req) {
setBasicAuthHeader(solrRequest, req);
for (HttpListenerFactory factory : listenerFactory) {
HttpListenerFactory.RequestResponseListener listener = factory.get();
req.onRequestQueued(listener);
req.onRequestBegin(listener);
req.onComplete(listener);
}
}
private Request createRequest(SolrRequest solrRequest, String collection) throws IOException, SolrServerException {
if (solrRequest.getBasePath() == null && serverBaseUrl == null)
throw new IllegalArgumentException("Destination node is not provided!");
if (solrRequest instanceof V2RequestSupport) {
solrRequest = ((V2RequestSupport) solrRequest).getV2Request();
}
SolrParams params = solrRequest.getParams();
RequestWriter.ContentWriter contentWriter = requestWriter.getContentWriter(solrRequest);
Collection<ContentStream> streams = contentWriter == null ? requestWriter.getContentStreams(solrRequest) : null;
String path = requestWriter.getPath(solrRequest);
if (path == null || !path.startsWith("/")) {
path = DEFAULT_PATH;
}
ResponseParser parser = solrRequest.getResponseParser();
if (parser == null) {
parser = this.parser;
}
// The parser 'wt=' and 'version=' params are used instead of the original
// params
ModifiableSolrParams wparams = new ModifiableSolrParams(params);
if (parser != null) {
wparams.set(CommonParams.WT, parser.getWriterType());
wparams.set(CommonParams.VERSION, parser.getVersion());
}
//TODO add invariantParams support
String basePath = solrRequest.getBasePath() == null ? serverBaseUrl : solrRequest.getBasePath();
if (collection != null)
basePath += "/" + collection;
if (solrRequest instanceof V2Request) {
if (System.getProperty("solr.v2RealPath") == null) {
basePath = serverBaseUrl.replace("/solr", "/api");
} else {
basePath = serverBaseUrl + "/____v2";
}
}
if (SolrRequest.METHOD.GET == solrRequest.getMethod()) {
if (streams != null || contentWriter != null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
}
return httpClient.newRequest(basePath + path + wparams.toQueryString()).method(HttpMethod.GET);
}
if (SolrRequest.METHOD.DELETE == solrRequest.getMethod()) {
return httpClient.newRequest(basePath + path + wparams.toQueryString()).method(HttpMethod.DELETE);
}
if (SolrRequest.METHOD.POST == solrRequest.getMethod() || SolrRequest.METHOD.PUT == solrRequest.getMethod()) {
String url = basePath + path;
boolean hasNullStreamName = false;
if (streams != null) {
hasNullStreamName = streams.stream().anyMatch(cs -> cs.getName() == null);
}
boolean isMultipart = streams != null && streams.size() > 1 && !hasNullStreamName;
HttpMethod method = SolrRequest.METHOD.POST == solrRequest.getMethod() ? HttpMethod.POST : HttpMethod.PUT;
if (contentWriter != null) {
Request req = httpClient
.newRequest(url + wparams.toQueryString())
.method(method);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
contentWriter.write(baos);
//TODO reduce memory usage
return req.content(new BytesContentProvider(contentWriter.getContentType(), baos.toByteArray()));
} else if (streams == null || isMultipart) {
// send server list and request list as query string params
ModifiableSolrParams queryParams = calculateQueryParams(this.queryParams, wparams);
queryParams.add(calculateQueryParams(solrRequest.getQueryParams(), wparams));
Request req = httpClient
.newRequest(url + queryParams.toQueryString())
.method(method);
return fillContentStream(req, streams, wparams, isMultipart);
} else {
// It is has one stream, it is the post body, put the params in the URL
ContentStream contentStream = streams.iterator().next();
return httpClient
.newRequest(url + wparams.toQueryString())
.method(method)
.content(new InputStreamContentProvider(contentStream.getStream()), contentStream.getContentType());
}
}
throw new SolrServerException("Unsupported method: " + solrRequest.getMethod());
}
private Request fillContentStream(Request req, Collection<ContentStream> streams,
ModifiableSolrParams wparams,
boolean isMultipart) throws IOException {
if (isMultipart) {
// multipart/form-data
MultiPartContentProvider content = new MultiPartContentProvider();
Iterator<String> iter = wparams.getParameterNamesIterator();
while (iter.hasNext()) {
String key = iter.next();
String[] vals = wparams.getParams(key);
if (vals != null) {
for (String val : vals) {
content.addFieldPart(key, new StringContentProvider(val), null);
}
}
}
if (streams != null) {
for (ContentStream contentStream : streams) {
String contentType = contentStream.getContentType();
if (contentType == null) {
contentType = BinaryResponseParser.BINARY_CONTENT_TYPE; // default
}
String name = contentStream.getName();
if (name == null) {
name = "";
}
HttpFields fields = new HttpFields();
fields.add(HttpHeader.CONTENT_TYPE, contentType);
content.addFilePart(name, contentStream.getName(), new InputStreamContentProvider(contentStream.getStream()), fields);
}
}
req.content(content);
} else {
// application/x-www-form-urlencoded
Fields fields = new Fields();
Iterator<String> iter = wparams.getParameterNamesIterator();
while (iter.hasNext()) {
String key = iter.next();
String[] vals = wparams.getParams(key);
if (vals != null) {
for (String val : vals) {
fields.add(key, val);
}
}
}
req.content(new FormContentProvider(fields, StandardCharsets.UTF_8));
}
return req;
}
private boolean wantStream(final ResponseParser processor) {
return processor == null || processor instanceof InputStreamResponseParser;
}
private NamedList<Object> processErrorsAndResponse(Response response,
final ResponseParser processor,
InputStream is,
String encoding,
final boolean isV2Api)
throws SolrServerException {
boolean shouldClose = true;
try {
// handle some http level checks before trying to parse the response
int httpStatus = response.getStatus();
String contentType;
contentType = response.getHeaders().get("content-type");
if (contentType == null) contentType = "";
switch (httpStatus) {
case HttpStatus.SC_OK:
case HttpStatus.SC_BAD_REQUEST:
case HttpStatus.SC_CONFLICT: // 409
break;
case HttpStatus.SC_MOVED_PERMANENTLY:
case HttpStatus.SC_MOVED_TEMPORARILY:
if (!httpClient.isFollowRedirects()) {
throw new SolrServerException("Server at " + getBaseURL()
+ " sent back a redirect (" + httpStatus + ").");
}
break;
default:
if (processor == null || "".equals(contentType)) {
throw new RemoteSolrException(serverBaseUrl, httpStatus, "non ok status: " + httpStatus
+ ", message:" + response.getReason(),
null);
}
}
if (wantStream(parser)) {
// no processor specified, return raw stream
NamedList<Object> rsp = new NamedList<>();
rsp.add("stream", is);
// Only case where stream should not be closed
shouldClose = false;
return rsp;
}
String procCt = processor.getContentType();
if (procCt != null) {
String procMimeType = ContentType.parse(procCt).getMimeType().trim().toLowerCase(Locale.ROOT);
String mimeType = ContentType.parse(contentType).getMimeType().trim().toLowerCase(Locale.ROOT);
if (!procMimeType.equals(mimeType)) {
// unexpected mime type
String msg = "Expected mime type " + procMimeType + " but got " + mimeType + ".";
try {
msg = msg + " " + IOUtils.toString(is, encoding);
} catch (IOException e) {
throw new RemoteSolrException(serverBaseUrl, httpStatus, "Could not parse response with encoding " + encoding, e);
}
throw new RemoteSolrException(serverBaseUrl, httpStatus, msg, null);
}
}
NamedList<Object> rsp;
try {
rsp = processor.processResponse(is, encoding);
} catch (Exception e) {
throw new RemoteSolrException(serverBaseUrl, httpStatus, e.getMessage(), e);
}
Object error = rsp == null ? null : rsp.get("error");
if (error != null && (String.valueOf(getObjectByPath(error, true, errPath)).endsWith("ExceptionWithErrObject"))) {
throw RemoteExecutionException.create(serverBaseUrl, rsp);
}
if (httpStatus != HttpStatus.SC_OK && !isV2Api) {
NamedList<String> metadata = null;
String reason = null;
try {
NamedList err = (NamedList) rsp.get("error");
if (err != null) {
reason = (String) err.get("msg");
if (reason == null) {
reason = (String) err.get("trace");
}
metadata = (NamedList<String>) err.get("metadata");
}
} catch (Exception ex) {}
if (reason == null) {
StringBuilder msg = new StringBuilder();
msg.append(response.getReason())
.append("\n\n")
.append("request: ")
.append(response.getRequest().getMethod());
try {
reason = java.net.URLDecoder.decode(msg.toString(), UTF_8);
} catch (UnsupportedEncodingException e) {
}
}
RemoteSolrException rss = new RemoteSolrException(serverBaseUrl, httpStatus, reason, null);
if (metadata != null) rss.setMetadata(metadata);
throw rss;
}
return rsp;
} finally {
if (shouldClose) {
try {
is.close();
assert ObjectReleaseTracker.release(is);
} catch (IOException e) {
// quitely
}
}
}
}
@Override
public NamedList<Object> request(SolrRequest request, String collection) throws SolrServerException, IOException {
return request(request, collection, null);
}
public void setRequestWriter(RequestWriter requestWriter) {
this.requestWriter = requestWriter;
}
public interface OnComplete {
void onSuccess(NamedList<Object> result);
void onFailure(Throwable e);
}
public void setFollowRedirects(boolean follow) {
httpClient.setFollowRedirects(follow);
}
public String getBaseURL() {
return serverBaseUrl;
}
private static class AsyncTracker {
private static final int MAX_OUTSTANDING_REQUESTS = 1000;
// wait for async requests
private final Phaser phaser;
// maximum outstanding requests left
private final Semaphore available;
private final Request.QueuedListener queuedListener;
private final Response.CompleteListener completeListener;
AsyncTracker() {
// TODO: what about shared instances?
phaser = new Phaser(1);
available = new Semaphore(MAX_OUTSTANDING_REQUESTS, false);
queuedListener = request -> {
phaser.register();
try {
available.acquire();
} catch (InterruptedException ignored) {
}
};
completeListener = result -> {
phaser.arriveAndDeregister();
available.release();
};
}
int getMaxRequestsQueuedPerDestination() {
// comfortably above max outstanding requests
return MAX_OUTSTANDING_REQUESTS * 3;
}
public void waitForComplete() {
phaser.arriveAndAwaitAdvance();
phaser.arriveAndDeregister();
}
}
public static class Builder {
private HttpClient httpClient;
private SSLConfig sslConfig = defaultSSLConfig;
private Integer idleTimeout;
private Integer connectionTimeout;
private Integer maxConnectionsPerHost;
private boolean useHttp1_1 = Boolean.getBoolean("solr.http1");
protected String baseSolrUrl;
public Builder() {
}
public Builder(String baseSolrUrl) {
this.baseSolrUrl = baseSolrUrl;
}
public Http2SolrClient build() {
return new Http2SolrClient(baseSolrUrl, this);
}
public Builder withHttpClient(HttpClient httpClient) {
this.httpClient = httpClient;
return this;
}
public Builder withSSLConfig(SSLConfig sslConfig) {
this.sslConfig = sslConfig;
return this;
}
/**
* Set maxConnectionsPerHost for http1 connections, maximum number http2 connections is limited by 4
*/
public Builder maxConnectionsPerHost(int max) {
this.maxConnectionsPerHost = max;
return this;
}
public Builder idleTimeout(int idleConnectionTimeout) {
this.idleTimeout = idleConnectionTimeout;
return this;
}
public Builder useHttp1_1(boolean useHttp1_1) {
this.useHttp1_1 = useHttp1_1;
return this;
}
public Builder connectionTimeout(int connectionTimeOut) {
this.connectionTimeout = connectionTimeOut;
return this;
}
}
/**
* Subclass of SolrException that allows us to capture an arbitrary HTTP status code that may have been returned by
* the remote server or a proxy along the way.
*/
public static class RemoteSolrException extends SolrException {
/**
* @param remoteHost the host the error was received from
* @param code Arbitrary HTTP status code
* @param msg Exception Message
* @param th Throwable to wrap with this Exception
*/
public RemoteSolrException(String remoteHost, int code, String msg, Throwable th) {
super(code, "Error from server at " + remoteHost + ": " + msg, th);
}
}
/**
* This should be thrown when a server has an error in executing the request and it sends a proper payload back to the
* client
*/
public static class RemoteExecutionException extends RemoteSolrException {
private NamedList meta;
public RemoteExecutionException(String remoteHost, int code, String msg, NamedList meta) {
super(remoteHost, code, msg, null);
this.meta = meta;
}
public static RemoteExecutionException create(String host, NamedList errResponse) {
Object errObj = errResponse.get("error");
if (errObj != null) {
Number code = (Number) getObjectByPath(errObj, true, Collections.singletonList("code"));
String msg = (String) getObjectByPath(errObj, true, Collections.singletonList("msg"));
return new RemoteExecutionException(host, code == null ? ErrorCode.UNKNOWN.code : code.intValue(),
msg == null ? "Unknown Error" : msg, errResponse);
} else {
throw new RuntimeException("No error");
}
}
public NamedList getMetaData() {
return meta;
}
}
public Set<String> getQueryParams() {
return queryParams;
}
/**
* Expert Method
*
* @param queryParams set of param keys to only send via the query string
* Note that the param will be sent as a query string if the key is part
* of this Set or the SolrRequest's query params.
* @see org.apache.solr.client.solrj.SolrRequest#getQueryParams
*/
public void setQueryParams(Set<String> queryParams) {
this.queryParams = queryParams;
}
private ModifiableSolrParams calculateQueryParams(Set<String> queryParamNames,
ModifiableSolrParams wparams) {
ModifiableSolrParams queryModParams = new ModifiableSolrParams();
if (queryParamNames != null) {
for (String param : queryParamNames) {
String[] value = wparams.getParams(param);
if (value != null) {
for (String v : value) {
queryModParams.add(param, v);
}
wparams.remove(param);
}
}
}
return queryModParams;
}
public ResponseParser getParser() {
return parser;
}
public void setParser(ResponseParser processor) {
parser = processor;
}
public static void setDefaultSSLConfig(SSLConfig sslConfig) {
Http2SolrClient.defaultSSLConfig = sslConfig;
}
// public for testing, only used by tests
public static void resetSslContextFactory() {
Http2SolrClient.defaultSSLConfig = null;
}
private static SslContextFactory getDefaultSslContextFactory() {
SslContextFactory sslContextFactory = new SslContextFactory(false);
if (null != System.getProperty("javax.net.ssl.keyStore")) {
sslContextFactory.setKeyStorePath
(System.getProperty("javax.net.ssl.keyStore"));
}
if (null != System.getProperty("javax.net.ssl.keyStorePassword")) {
sslContextFactory.setKeyStorePassword
(System.getProperty("javax.net.ssl.keyStorePassword"));
}
if (null != System.getProperty("javax.net.ssl.trustStore")) {
sslContextFactory.setTrustStorePath
(System.getProperty("javax.net.ssl.trustStore"));
}
if (null != System.getProperty("javax.net.ssl.trustStorePassword")) {
sslContextFactory.setTrustStorePassword
(System.getProperty("javax.net.ssl.trustStorePassword"));
}
String checkPeerNameStr = System.getProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME);
boolean sslCheckPeerName = true;
if (checkPeerNameStr == null || "false".equalsIgnoreCase(checkPeerNameStr)) {
sslCheckPeerName = false;
}
if (System.getProperty("tests.jettySsl.clientAuth") != null) {
sslCheckPeerName = sslCheckPeerName || Boolean.getBoolean("tests.jettySsl.clientAuth");
}
sslContextFactory.setNeedClientAuth(sslCheckPeerName);
return sslContextFactory;
}
}

3
solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClientBuilderFactory.java
View File

@ -38,4 +38,7 @@ public interface HttpClientBuilderFactory extends Closeable {
*/
public SolrHttpClientBuilder getHttpClientBuilder(Optional<SolrHttpClientBuilder> builder);
public default void setup(Http2SolrClient client) {
}
}

38
solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpListenerFactory.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import org.eclipse.jetty.client.api.Request;
import org.eclipse.jetty.client.api.Response;
import org.eclipse.jetty.client.api.Result;
public interface HttpListenerFactory {
abstract class RequestResponseListener implements Request.BeginListener, Response.CompleteListener, Request.QueuedListener {
@Override
public void onBegin(Request request){}
@Override
public void onQueued(Request request) {}
@Override
public void onComplete(Result result) {}
}
RequestResponseListener get();
}

60
solr/solrj/src/java/org/apache/solr/client/solrj/impl/Krb5HttpClientBuilder.java
View File

@ -16,17 +16,19 @@
*/
package org.apache.solr.client.solrj.impl;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.Configuration;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.nio.file.Paths;
import java.security.Principal;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.Configuration;
import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpRequestInterceptor;
@ -41,6 +43,9 @@ import org.apache.http.cookie.CookieSpecProvider;
import org.apache.http.entity.BufferedHttpEntity;
import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.eclipse.jetty.client.HttpAuthenticationStore;
import org.eclipse.jetty.client.WWWAuthenticationProtocolHandler;
import org.eclipse.jetty.client.util.SPNEGOAuthentication;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -50,8 +55,9 @@ import org.slf4j.LoggerFactory;
public class Krb5HttpClientBuilder implements HttpClientBuilderFactory {
public static final String LOGIN_CONFIG_PROP = "java.security.auth.login.config";
private static final String SPNEGO_OID = "1.3.6.1.5.5.2";
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static Configuration jaasConfig = new SolrJaasConfiguration();
public Krb5HttpClientBuilder() {
@ -80,6 +86,49 @@ public class Krb5HttpClientBuilder implements HttpClientBuilderFactory {
return builder.isPresent() ? getBuilder(builder.get()) : getBuilder();
}
private SPNEGOAuthentication createSPNEGOAuthentication() {
SPNEGOAuthentication authentication = new SPNEGOAuthentication(null){
public boolean matches(String type, URI uri, String realm) {
return this.getType().equals(type);
}
};
String clientAppName = System.getProperty("solr.kerberos.jaas.appname", "Client");
AppConfigurationEntry[] entries = jaasConfig.getAppConfigurationEntry(clientAppName);
if (entries == null) {
log.warn("Could not find login configuration entry for {}. SPNego authentication may not be successful.", (Object)clientAppName);
return authentication;
}
if (entries.length != 1) {
log.warn("Multiple login modules are specified in the configuration file");
return authentication;
}
Map<String, ?> options = entries[0].getOptions();
String keyTab = (String)options.get("keyTab");
if (keyTab != null) {
authentication.setUserKeyTabPath(Paths.get(keyTab, new String[0]));
}
authentication.setServiceName("HTTP");
authentication.setUserName((String)options.get("principal"));
if ("true".equalsIgnoreCase((String)options.get("useTicketCache"))) {
authentication.setUseTicketCache(true);
String ticketCachePath = (String)options.get("ticketCache");
if (ticketCachePath != null) {
authentication.setTicketCachePath(Paths.get(ticketCachePath));
}
authentication.setRenewTGT("true".equalsIgnoreCase((String)options.get("renewTGT")));
}
return authentication;
}
@Override
public void setup(Http2SolrClient http2Client) {
HttpAuthenticationStore authenticationStore = new HttpAuthenticationStore();
authenticationStore.addAuthentication(createSPNEGOAuthentication());
http2Client.getHttpClient().setAuthenticationStore(authenticationStore);
http2Client.getProtocolHandlers().put(new WWWAuthenticationProtocolHandler(http2Client.getHttpClient()));
}
public SolrHttpClientBuilder getBuilder(SolrHttpClientBuilder builder) {
if (System.getProperty(LOGIN_CONFIG_PROP) != null) {
String configValue = System.getProperty(LOGIN_CONFIG_PROP);
@ -93,8 +142,7 @@ public class Krb5HttpClientBuilder implements HttpClientBuilderFactory {
// authentication mechanism can load the credentials from the JAAS configuration.
if (useSubjectCredsVal == null) {
System.setProperty(useSubjectCredsProp, "false");
}
else if (!useSubjectCredsVal.toLowerCase(Locale.ROOT).equals("false")) {
} else if (!useSubjectCredsVal.toLowerCase(Locale.ROOT).equals("false")) {
// Don't overwrite the prop value if it's already been written to something else,
// but log because it is likely the Credentials won't be loaded correctly.
log.warn("System Property: " + useSubjectCredsProp + " set to: " + useSubjectCredsVal

69
solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import java.util.Arrays;
import org.apache.solr.client.solrj.SolrClient;
/**
* LBHttp2SolrClient or "LoadBalanced LBHttp2SolrClient" is a load balancing wrapper around
* {@link Http2SolrClient}. This is useful when you
* have multiple Solr servers and the requests need to be Load Balanced among them.
*
* Do <b>NOT</b> use this class for indexing in master/slave scenarios since documents must be sent to the
* correct master; no inter-node routing is done.
*
* In SolrCloud (leader/replica) scenarios, it is usually better to use
* {@link CloudSolrClient}, but this class may be used
* for updates because the server will forward them to the appropriate leader.
*
* <p>
* It offers automatic failover when a server goes down and it detects when the server comes back up.
* <p>
* Load balancing is done using a simple round-robin on the list of servers.
* <p>
* If a request to a server fails by an IOException due to a connection timeout or read timeout then the host is taken
* off the list of live servers and moved to a 'dead server list' and the request is resent to the next live server.
* This process is continued till it tries all the live servers. If at least one server is alive, the request succeeds,
* and if not it fails.
* <blockquote><pre>
* SolrClient lbHttp2SolrClient = new LBHttp2SolrClient(http2SolrClient, "http://host1:8080/solr/", "http://host2:8080/solr", "http://host2:8080/solr");
* </pre></blockquote>
* This detects if a dead server comes alive automatically. The check is done in fixed intervals in a dedicated thread.
* This interval can be set using {@link #setAliveCheckInterval} , the default is set to one minute.
* <p>
* <b>When to use this?</b><br> This can be used as a software load balancer when you do not wish to setup an external
* load balancer. Alternatives to this code are to use
* a dedicated hardware load balancer or using Apache httpd with mod_proxy_balancer as a load balancer. See <a
* href="http://en.wikipedia.org/wiki/Load_balancing_(computing)">Load balancing on Wikipedia</a>
*
* @lucene.experimental
* @since solr 8.0
*/
public class LBHttp2SolrClient extends LBSolrClient {
private Http2SolrClient httpClient;
public LBHttp2SolrClient(Http2SolrClient httpClient, String... baseSolrUrls) {
super(Arrays.asList(baseSolrUrls));
this.httpClient = httpClient;
}
@Override
protected SolrClient getClient(String baseUrl) {
return httpClient;
}
}

720
solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java
View File

@ -17,46 +17,16 @@
package org.apache.solr.client.solrj.impl;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.net.ConnectException;
import java.net.MalformedURLException;
import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.ResponseParser;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteExecutionException;
import org.apache.solr.client.solrj.request.IsUpdateRequest;
import org.apache.solr.client.solrj.request.RequestWriter;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.slf4j.MDC;
import static org.apache.solr.common.params.CommonParams.ADMIN_PATHS;
/**
* LBHttpSolrClient or "LoadBalanced HttpSolrClient" is a load balancing wrapper around
@ -95,147 +65,36 @@ import static org.apache.solr.common.params.CommonParams.ADMIN_PATHS;
*
* @since solr 1.4
*/
public class LBHttpSolrClient extends SolrClient {
private static Set<Integer> RETRY_CODES = new HashSet<>(4);
static {
RETRY_CODES.add(404);
RETRY_CODES.add(403);
RETRY_CODES.add(503);
RETRY_CODES.add(500);
}
// keys to the maps are currently of the form "http://localhost:8983/solr"
// which should be equivalent to HttpSolrServer.getBaseURL()
private final Map<String, ServerWrapper> aliveServers = new LinkedHashMap<>();
// access to aliveServers should be synchronized on itself
protected final Map<String, ServerWrapper> zombieServers = new ConcurrentHashMap<>();
// changes to aliveServers are reflected in this array, no need to synchronize
private volatile ServerWrapper[] aliveServerList = new ServerWrapper[0];
private volatile ScheduledExecutorService aliveCheckExecutor;
public class LBHttpSolrClient extends LBSolrClient {
private final HttpClient httpClient;
private final boolean clientIsInternal;
private final ConcurrentHashMap<String, HttpSolrClient> urlToClient = new ConcurrentHashMap<>();
private final HttpSolrClient.Builder httpSolrClientBuilder;
private final AtomicInteger counter = new AtomicInteger(-1);
private static final SolrQuery solrQuery = new SolrQuery("*:*");
private volatile ResponseParser parser;
private volatile RequestWriter requestWriter;
private Set<String> queryParams = new HashSet<>();
private Integer connectionTimeout;
private volatile Integer soTimeout;
static {
solrQuery.setRows(0);
/**
* Default sort (if we don't supply a sort) is by score and since
* we request 0 rows any sorting and scoring is not necessary.
* SolrQuery.DOCID schema-independently specifies a non-scoring sort.
* <code>_docid_ asc</code> sort is efficient,
* <code>_docid_ desc</code> sort is not, so choose ascending DOCID sort.
*/
solrQuery.setSort(SolrQuery.DOCID, SolrQuery.ORDER.asc);
// not a top-level request, we are interested only in the server being sent to i.e. it need not distribute our request to further servers
solrQuery.setDistrib(false);
}
protected static class ServerWrapper {
final HttpSolrClient client;
// "standard" servers are used by default. They normally live in the alive list
// and move to the zombie list when unavailable. When they become available again,
// they move back to the alive list.
boolean standard = true;
int failedPings = 0;
public ServerWrapper(HttpSolrClient client) {
this.client = client;
}
@Override
public String toString() {
return client.getBaseURL();
}
public String getKey() {
return client.getBaseURL();
}
@Override
public int hashCode() {
return this.getKey().hashCode();
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (!(obj instanceof ServerWrapper)) return false;
return this.getKey().equals(((ServerWrapper)obj).getKey());
}
}
public static class Req {
protected SolrRequest request;
protected List<String> servers;
protected int numDeadServersToTry;
private final Integer numServersToTry;
/**
* @deprecated use {@link LBSolrClient.Req} instead
*/
@Deprecated
public static class Req extends LBSolrClient.Req {
public Req(SolrRequest request, List<String> servers) {
this(request, servers, null);
super(request, servers);
}
public Req(SolrRequest request, List<String> servers, Integer numServersToTry) {
this.request = request;
this.servers = servers;
this.numDeadServersToTry = servers.size();
this.numServersToTry = numServersToTry;
}
public SolrRequest getRequest() {
return request;
}
public List<String> getServers() {
return servers;
}
/** @return the number of dead servers to try if there are no live servers left */
public int getNumDeadServersToTry() {
return numDeadServersToTry;
}
/** @param numDeadServersToTry The number of dead servers to try if there are no live servers left.
* Defaults to the number of servers in this request. */
public void setNumDeadServersToTry(int numDeadServersToTry) {
this.numDeadServersToTry = numDeadServersToTry;
}
public Integer getNumServersToTry() {
return numServersToTry;
super(request, servers, numServersToTry);
}
}
public static class Rsp {
protected String server;
protected NamedList<Object> rsp;
/**
* @deprecated use {@link LBSolrClient.Rsp} instead
*/
@Deprecated
public static class Rsp extends LBSolrClient.Rsp {
/** The response from the server */
public NamedList<Object> getResponse() {
return rsp;
}
/** The server that returned the response */
public String getServer() {
return server;
}
}
/**
@ -266,20 +125,16 @@ public class LBHttpSolrClient extends SolrClient {
}
protected LBHttpSolrClient(Builder builder) {
super(builder.baseSolrUrls);
this.clientIsInternal = builder.httpClient == null;
this.httpSolrClientBuilder = builder.httpSolrClientBuilder;
this.httpClient = builder.httpClient == null ? constructClient(builder.baseSolrUrls.toArray(new String[builder.baseSolrUrls.size()])) : builder.httpClient;
this.connectionTimeout = builder.connectionTimeoutMillis;
this.soTimeout = builder.socketTimeoutMillis;
this.parser = builder.responseParser;
if (! builder.baseSolrUrls.isEmpty()) {
for (String s : builder.baseSolrUrls) {
ServerWrapper wrapper = new ServerWrapper(makeSolrClient(s));
aliveServers.put(wrapper.getKey(), wrapper);
}
for (String baseUrl: builder.baseSolrUrls) {
urlToClient.put(baseUrl, makeSolrClient(baseUrl));
}
updateAliveList();
}
private HttpClient constructClient(String[] solrServerUrl) {
@ -293,27 +148,6 @@ public class LBHttpSolrClient extends SolrClient {
return HttpClientUtil.createClient(params);
}
public Set<String> getQueryParams() {
return queryParams;
}
/**
* Expert Method.
* @param queryParams set of param keys to only send via the query string
*/
public void setQueryParams(Set<String> queryParams) {
this.queryParams = queryParams;
}
public void addQueryParams(String queryOnlyParam) {
this.queryParams.add(queryOnlyParam) ;
}
public static String normalize(String server) {
if (server.endsWith("/"))
server = server.substring(0, server.length() - 1);
return server;
}
protected HttpSolrClient makeSolrClient(String server) {
HttpSolrClient client;
if (httpSolrClientBuilder != null) {
@ -350,244 +184,13 @@ public class LBHttpSolrClient extends SolrClient {
return client;
}
/**
* Tries to query a live server from the list provided in Req. Servers in the dead pool are skipped.
* If a request fails due to an IOException, the server is moved to the dead pool for a certain period of
* time, or until a test request on that server succeeds.
*
* Servers are queried in the exact order given (except servers currently in the dead pool are skipped).
* If no live servers from the provided list remain to be tried, a number of previously skipped dead servers will be tried.
* Req.getNumDeadServersToTry() controls how many dead servers will be tried.
*
* If no live servers are found a SolrServerException is thrown.
*
* @param req contains both the request as well as the list of servers to query
*
* @return the result of the request
*
* @throws IOException If there is a low-level I/O error.
*/
public Rsp request(Req req) throws SolrServerException, IOException {
Rsp rsp = new Rsp();
Exception ex = null;
boolean isNonRetryable = req.request instanceof IsUpdateRequest || ADMIN_PATHS.contains(req.request.getPath());
List<ServerWrapper> skipped = null;
final Integer numServersToTry = req.getNumServersToTry();
int numServersTried = 0;
boolean timeAllowedExceeded = false;
long timeAllowedNano = getTimeAllowedInNanos(req.getRequest());
long timeOutTime = System.nanoTime() + timeAllowedNano;
for (String serverStr : req.getServers()) {
if (timeAllowedExceeded = isTimeExceeded(timeAllowedNano, timeOutTime)) {
break;
}
serverStr = normalize(serverStr);
// if the server is currently a zombie, just skip to the next one
ServerWrapper wrapper = zombieServers.get(serverStr);
if (wrapper != null) {
// System.out.println("ZOMBIE SERVER QUERIED: " + serverStr);
final int numDeadServersToTry = req.getNumDeadServersToTry();
if (numDeadServersToTry > 0) {
if (skipped == null) {
skipped = new ArrayList<>(numDeadServersToTry);
skipped.add(wrapper);
}
else if (skipped.size() < numDeadServersToTry) {
skipped.add(wrapper);
}
}
continue;
}
try {
MDC.put("LBHttpSolrClient.url", serverStr);
if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
break;
}
HttpSolrClient client = makeSolrClient(serverStr);
++numServersTried;
ex = doRequest(client, req, rsp, isNonRetryable, false, null);
if (ex == null) {
return rsp; // SUCCESS
}
} finally {
MDC.remove("LBHttpSolrClient.url");
}
}
// try the servers we previously skipped
if (skipped != null) {
for (ServerWrapper wrapper : skipped) {
if (timeAllowedExceeded = isTimeExceeded(timeAllowedNano, timeOutTime)) {
break;
}
if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
break;
}
try {
MDC.put("LBHttpSolrClient.url", wrapper.client.getBaseURL());
++numServersTried;
ex = doRequest(wrapper.client, req, rsp, isNonRetryable, true, wrapper.getKey());
if (ex == null) {
return rsp; // SUCCESS
}
} finally {
MDC.remove("LBHttpSolrClient.url");
}
}
}
final String solrServerExceptionMessage;
if (timeAllowedExceeded) {
solrServerExceptionMessage = "Time allowed to handle this request exceeded";
} else {
if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
solrServerExceptionMessage = "No live SolrServers available to handle this request:"
+ " numServersTried="+numServersTried
+ " numServersToTry="+numServersToTry.intValue();
} else {
solrServerExceptionMessage = "No live SolrServers available to handle this request";
}
}
if (ex == null) {
throw new SolrServerException(solrServerExceptionMessage);
} else {
throw new SolrServerException(solrServerExceptionMessage+":" + zombieServers.keySet(), ex);
}
}
protected Exception addZombie(HttpSolrClient server, Exception e) {
ServerWrapper wrapper;
wrapper = new ServerWrapper(server);
wrapper.standard = false;
zombieServers.put(wrapper.getKey(), wrapper);
startAliveCheckExecutor();
return e;
}
protected Exception doRequest(HttpSolrClient client, Req req, Rsp rsp, boolean isNonRetryable,
boolean isZombie, String zombieKey) throws SolrServerException, IOException {
Exception ex = null;
try {
rsp.server = client.getBaseURL();
rsp.rsp = client.request(req.getRequest(), (String) null);
if (isZombie) {
zombieServers.remove(zombieKey);
}
} catch (RemoteExecutionException e){
throw e;
} catch(SolrException e) {
// we retry on 404 or 403 or 503 or 500
// unless it's an update - then we only retry on connect exception
if (!isNonRetryable && RETRY_CODES.contains(e.code())) {
ex = (!isZombie) ? addZombie(client, e) : e;
} else {
// Server is alive but the request was likely malformed or invalid
if (isZombie) {
zombieServers.remove(zombieKey);
}
throw e;
}
} catch (SocketException e) {
if (!isNonRetryable || e instanceof ConnectException) {
ex = (!isZombie) ? addZombie(client, e) : e;
} else {
throw e;
}
} catch (SocketTimeoutException e) {
if (!isNonRetryable) {
ex = (!isZombie) ? addZombie(client, e) : e;
} else {
throw e;
}
} catch (SolrServerException e) {
Throwable rootCause = e.getRootCause();
if (!isNonRetryable && rootCause instanceof IOException) {
ex = (!isZombie) ? addZombie(client, e) : e;
} else if (isNonRetryable && rootCause instanceof ConnectException) {
ex = (!isZombie) ? addZombie(client, e) : e;
} else {
throw e;
}
} catch (Exception e) {
throw new SolrServerException(e);
}
return ex;
}
private void updateAliveList() {
synchronized (aliveServers) {
aliveServerList = aliveServers.values().toArray(new ServerWrapper[aliveServers.size()]);
}
}
private ServerWrapper removeFromAlive(String key) {
synchronized (aliveServers) {
ServerWrapper wrapper = aliveServers.remove(key);
if (wrapper != null)
updateAliveList();
return wrapper;
}
}
private void addToAlive(ServerWrapper wrapper) {
synchronized (aliveServers) {
ServerWrapper prev = aliveServers.put(wrapper.getKey(), wrapper);
// TODO: warn if there was a previous entry?
updateAliveList();
}
}
public void addSolrServer(String server) throws MalformedURLException {
HttpSolrClient client = makeSolrClient(server);
addToAlive(new ServerWrapper(client));
}
public String removeSolrServer(String server) {
try {
server = new URL(server).toExternalForm();
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
if (server.endsWith("/")) {
server = server.substring(0, server.length() - 1);
}
// there is a small race condition here - if the server is in the process of being moved between
// lists, we could fail to remove it.
removeFromAlive(server);
zombieServers.remove(server);
return null;
}
/**
* @deprecated since 7.0 Use {@link Builder} methods instead.
*/
@Deprecated
public void setConnectionTimeout(int timeout) {
this.connectionTimeout = timeout;
synchronized (aliveServers) {
Iterator<ServerWrapper> wrappersIt = aliveServers.values().iterator();
while (wrappersIt.hasNext()) {
wrappersIt.next().client.setConnectionTimeout(timeout);
}
}
Iterator<ServerWrapper> wrappersIt = zombieServers.values().iterator();
while (wrappersIt.hasNext()) {
wrappersIt.next().client.setConnectionTimeout(timeout);
}
this.urlToClient.values().forEach(client -> client.setConnectionTimeout(timeout));
}
/**
@ -599,239 +202,46 @@ public class LBHttpSolrClient extends SolrClient {
@Deprecated
public void setSoTimeout(int timeout) {
this.soTimeout = timeout;
synchronized (aliveServers) {
Iterator<ServerWrapper> wrappersIt = aliveServers.values().iterator();
while (wrappersIt.hasNext()) {
wrappersIt.next().client.setSoTimeout(timeout);
}
}
Iterator<ServerWrapper> wrappersIt = zombieServers.values().iterator();
while (wrappersIt.hasNext()) {
wrappersIt.next().client.setSoTimeout(timeout);
this.urlToClient.values().forEach(client -> client.setSoTimeout(timeout));
}
/**
* @deprecated use {@link LBSolrClient#request(LBSolrClient.Req)} instead
*/
@Deprecated
public Rsp request(Req req) throws SolrServerException, IOException {
LBSolrClient.Rsp rsp = super.request(req);
// for backward-compatibility support
Rsp result = new Rsp();
result.rsp = rsp.rsp;
result.server = rsp.server;
return result;
}
@Override
protected SolrClient getClient(String baseUrl) {
HttpSolrClient client = urlToClient.get(baseUrl);
if (client == null) {
return makeSolrClient(baseUrl);
} else {
return client;
}
}
@Override
public String removeSolrServer(String server) {
urlToClient.remove(server);
return super.removeSolrServer(server);
}
@Override
public void close() {
synchronized (this) {
if (aliveCheckExecutor != null) {
aliveCheckExecutor.shutdownNow();
ExecutorUtil.shutdownAndAwaitTermination(aliveCheckExecutor);
}
}
super.close();
if(clientIsInternal) {
HttpClientUtil.close(httpClient);
}
}
/**
* Tries to query a live server. A SolrServerException is thrown if all servers are dead.
* If the request failed due to IOException then the live server is moved to dead pool and the request is
* retried on another live server. After live servers are exhausted, any servers previously marked as dead
* will be tried before failing the request.
*
* @param request the SolrRequest.
*
* @return response
*
* @throws IOException If there is a low-level I/O error.
*/
@Override
public NamedList<Object> request(final SolrRequest request, String collection)
throws SolrServerException, IOException {
return request(request, collection, null);
}
public NamedList<Object> request(final SolrRequest request, String collection,
final Integer numServersToTry) throws SolrServerException, IOException {
Exception ex = null;
ServerWrapper[] serverList = aliveServerList;
final int maxTries = (numServersToTry == null ? serverList.length : numServersToTry.intValue());
int numServersTried = 0;
Map<String,ServerWrapper> justFailed = null;
boolean timeAllowedExceeded = false;
long timeAllowedNano = getTimeAllowedInNanos(request);
long timeOutTime = System.nanoTime() + timeAllowedNano;
for (int attempts=0; attempts<maxTries; attempts++) {
if (timeAllowedExceeded = isTimeExceeded(timeAllowedNano, timeOutTime)) {
break;
}
int count = counter.incrementAndGet() & Integer.MAX_VALUE;
ServerWrapper wrapper = serverList[count % serverList.length];
try {
++numServersTried;
return wrapper.client.request(request, collection);
} catch (SolrException e) {
// Server is alive but the request was malformed or invalid
throw e;
} catch (SolrServerException e) {
if (e.getRootCause() instanceof IOException) {
ex = e;
moveAliveToDead(wrapper);
if (justFailed == null) justFailed = new HashMap<>();
justFailed.put(wrapper.getKey(), wrapper);
} else {
throw e;
}
} catch (Exception e) {
throw new SolrServerException(e);
}
}
// try other standard servers that we didn't try just now
for (ServerWrapper wrapper : zombieServers.values()) {
if (timeAllowedExceeded = isTimeExceeded(timeAllowedNano, timeOutTime)) {
break;
}
if (wrapper.standard==false || justFailed!=null && justFailed.containsKey(wrapper.getKey())) continue;
try {
++numServersTried;
NamedList<Object> rsp = wrapper.client.request(request, collection);
// remove from zombie list *before* adding to alive to avoid a race that could lose a server
zombieServers.remove(wrapper.getKey());
addToAlive(wrapper);
return rsp;
} catch (SolrException e) {
// Server is alive but the request was malformed or invalid
throw e;
} catch (SolrServerException e) {
if (e.getRootCause() instanceof IOException) {
ex = e;
// still dead
} else {
throw e;
}
} catch (Exception e) {
throw new SolrServerException(e);
}
}
final String solrServerExceptionMessage;
if (timeAllowedExceeded) {
solrServerExceptionMessage = "Time allowed to handle this request exceeded";
} else {
if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
solrServerExceptionMessage = "No live SolrServers available to handle this request:"
+ " numServersTried="+numServersTried
+ " numServersToTry="+numServersToTry.intValue();
} else {
solrServerExceptionMessage = "No live SolrServers available to handle this request";
}
}
if (ex == null) {
throw new SolrServerException(solrServerExceptionMessage);
} else {
throw new SolrServerException(solrServerExceptionMessage, ex);
}
}
/**
* @return time allowed in nanos, returns -1 if no time_allowed is specified.
*/
private long getTimeAllowedInNanos(final SolrRequest req) {
SolrParams reqParams = req.getParams();
return reqParams == null ? -1 :
TimeUnit.NANOSECONDS.convert(reqParams.getInt(CommonParams.TIME_ALLOWED, -1), TimeUnit.MILLISECONDS);
}
private boolean isTimeExceeded(long timeAllowedNano, long timeOutTime) {
return timeAllowedNano > 0 && System.nanoTime() > timeOutTime;
}
/**
* Takes up one dead server and check for aliveness. The check is done in a roundrobin. Each server is checked for
* aliveness once in 'x' millis where x is decided by the setAliveCheckinterval() or it is defaulted to 1 minute
*
* @param zombieServer a server in the dead pool
*/
private void checkAZombieServer(ServerWrapper zombieServer) {
try {
QueryResponse resp = zombieServer.client.query(solrQuery);
if (resp.getStatus() == 0) {
// server has come back up.
// make sure to remove from zombies before adding to alive to avoid a race condition
// where another thread could mark it down, move it back to zombie, and then we delete
// from zombie and lose it forever.
ServerWrapper wrapper = zombieServers.remove(zombieServer.getKey());
if (wrapper != null) {
wrapper.failedPings = 0;
if (wrapper.standard) {
addToAlive(wrapper);
}
} else {
// something else already moved the server from zombie to alive
}
}
} catch (Exception e) {
//Expected. The server is still down.
zombieServer.failedPings++;
// If the server doesn't belong in the standard set belonging to this load balancer
// then simply drop it after a certain number of failed pings.
if (!zombieServer.standard && zombieServer.failedPings >= NONSTANDARD_PING_LIMIT) {
zombieServers.remove(zombieServer.getKey());
}
}
}
private void moveAliveToDead(ServerWrapper wrapper) {
wrapper = removeFromAlive(wrapper.getKey());
if (wrapper == null)
return; // another thread already detected the failure and removed it
zombieServers.put(wrapper.getKey(), wrapper);
startAliveCheckExecutor();
}
private int interval = CHECK_INTERVAL;
/**
* LBHttpSolrServer keeps pinging the dead servers at fixed interval to find if it is alive. Use this to set that
* interval
*
* @param interval time in milliseconds
*/
public void setAliveCheckInterval(int interval) {
if (interval <= 0) {
throw new IllegalArgumentException("Alive check interval must be " +
"positive, specified value = " + interval);
}
this.interval = interval;
}
private void startAliveCheckExecutor() {
// double-checked locking, but it's OK because we don't *do* anything with aliveCheckExecutor
// if it's not null.
if (aliveCheckExecutor == null) {
synchronized (this) {
if (aliveCheckExecutor == null) {
aliveCheckExecutor = Executors.newSingleThreadScheduledExecutor(
new SolrjNamedThreadFactory("aliveCheckExecutor"));
aliveCheckExecutor.scheduleAtFixedRate(
getAliveCheckRunner(new WeakReference<>(this)),
this.interval, this.interval, TimeUnit.MILLISECONDS);
}
}
}
}
private static Runnable getAliveCheckRunner(final WeakReference<LBHttpSolrClient> lbRef) {
return () -> {
LBHttpSolrClient lb = lbRef.get();
if (lb != null && lb.zombieServers != null) {
for (ServerWrapper zombieServer : lb.zombieServers.values()) {
lb.checkAZombieServer(zombieServer);
}
}
};
}
/**
* Return the HttpClient this instance uses.
*/
@ -839,40 +249,6 @@ public class LBHttpSolrClient extends SolrClient {
return httpClient;
}
public ResponseParser getParser() {
return parser;
}
/**
* Changes the {@link ResponseParser} that will be used for the internal
* SolrServer objects.
*
* @param parser Default Response Parser chosen to parse the response if the parser
* were not specified as part of the request.
* @see org.apache.solr.client.solrj.SolrRequest#getResponseParser()
*/
public void setParser(ResponseParser parser) {
this.parser = parser;
}
/**
* Changes the {@link RequestWriter} that will be used for the internal
* SolrServer objects.
*
* @param requestWriter Default RequestWriter, used to encode requests sent to the server.
*/
public void setRequestWriter(RequestWriter requestWriter) {
this.requestWriter = requestWriter;
}
public RequestWriter getRequestWriter() {
return requestWriter;
}
// defaults
private static final int CHECK_INTERVAL = 60 * 1000; //1 minute between checks
private static final int NONSTANDARD_PING_LIMIT = 5; // number of times we'll ping dead servers not in the server list
/**
* Constructs {@link LBHttpSolrClient} instances from provided configuration.
*/

703
solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBSolrClient.java Normal file
View File

@ -0,0 +1,703 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import java.io.IOException;
import java.lang.ref.WeakReference;
import java.net.ConnectException;
import java.net.MalformedURLException;
import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.solr.client.solrj.ResponseParser;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.IsUpdateRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.RequestWriter;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.slf4j.MDC;
import static org.apache.solr.common.params.CommonParams.ADMIN_PATHS;
public abstract class LBSolrClient extends SolrClient {
// defaults
private static final Set<Integer> RETRY_CODES = new HashSet<>(Arrays.asList(404, 403, 503, 500));
private static final int CHECK_INTERVAL = 60 * 1000; //1 minute between checks
private static final int NONSTANDARD_PING_LIMIT = 5; // number of times we'll ping dead servers not in the server list
// keys to the maps are currently of the form "http://localhost:8983/solr"
// which should be equivalent to HttpSolrServer.getBaseURL()
private final Map<String, ServerWrapper> aliveServers = new LinkedHashMap<>();
// access to aliveServers should be synchronized on itself
private final Map<String, ServerWrapper> zombieServers = new ConcurrentHashMap<>();
// changes to aliveServers are reflected in this array, no need to synchronize
private volatile ServerWrapper[] aliveServerList = new ServerWrapper[0];
private volatile ScheduledExecutorService aliveCheckExecutor;
private int interval = CHECK_INTERVAL;
private final AtomicInteger counter = new AtomicInteger(-1);
private static final SolrQuery solrQuery = new SolrQuery("*:*");
protected volatile ResponseParser parser;
protected volatile RequestWriter requestWriter;
protected Set<String> queryParams = new HashSet<>();
static {
solrQuery.setRows(0);
/**
* Default sort (if we don't supply a sort) is by score and since
* we request 0 rows any sorting and scoring is not necessary.
* SolrQuery.DOCID schema-independently specifies a non-scoring sort.
* <code>_docid_ asc</code> sort is efficient,
* <code>_docid_ desc</code> sort is not, so choose ascending DOCID sort.
*/
solrQuery.setSort(SolrQuery.DOCID, SolrQuery.ORDER.asc);
// not a top-level request, we are interested only in the server being sent to i.e. it need not distribute our request to further servers
solrQuery.setDistrib(false);
}
protected static class ServerWrapper {
final String baseUrl;
// "standard" servers are used by default. They normally live in the alive list
// and move to the zombie list when unavailable. When they become available again,
// they move back to the alive list.
boolean standard = true;
int failedPings = 0;
ServerWrapper(String baseUrl) {
this.baseUrl = baseUrl;
}
public String getBaseUrl() {
return baseUrl;
}
@Override
public String toString() {
return baseUrl;
}
@Override
public int hashCode() {
return baseUrl.hashCode();
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (!(obj instanceof ServerWrapper)) return false;
return baseUrl.equals(((ServerWrapper)obj).baseUrl);
}
}
public static class Req {
protected SolrRequest request;
protected List<String> servers;
protected int numDeadServersToTry;
private final Integer numServersToTry;
public Req(SolrRequest request, List<String> servers) {
this(request, servers, null);
}
public Req(SolrRequest request, List<String> servers, Integer numServersToTry) {
this.request = request;
this.servers = servers;
this.numDeadServersToTry = servers.size();
this.numServersToTry = numServersToTry;
}
public SolrRequest getRequest() {
return request;
}
public List<String> getServers() {
return servers;
}
/** @return the number of dead servers to try if there are no live servers left */
public int getNumDeadServersToTry() {
return numDeadServersToTry;
}
/** @param numDeadServersToTry The number of dead servers to try if there are no live servers left.
* Defaults to the number of servers in this request. */
public void setNumDeadServersToTry(int numDeadServersToTry) {
this.numDeadServersToTry = numDeadServersToTry;
}
public Integer getNumServersToTry() {
return numServersToTry;
}
}
public static class Rsp {
protected String server;
protected NamedList<Object> rsp;
/** The response from the server */
public NamedList<Object> getResponse() {
return rsp;
}
/** The server that returned the response */
public String getServer() {
return server;
}
}
public LBSolrClient(List<String> baseSolrUrls) {
if (!baseSolrUrls.isEmpty()) {
for (String s : baseSolrUrls) {
ServerWrapper wrapper = createServerWrapper(s);
aliveServers.put(wrapper.getBaseUrl(), wrapper);
}
updateAliveList();
}
}
protected void updateAliveList() {
synchronized (aliveServers) {
aliveServerList = aliveServers.values().toArray(new ServerWrapper[0]);
}
}
protected ServerWrapper createServerWrapper(String baseUrl) {
return new ServerWrapper(baseUrl);
}
public Set<String> getQueryParams() {
return queryParams;
}
/**
* Expert Method.
* @param queryParams set of param keys to only send via the query string
*/
public void setQueryParams(Set<String> queryParams) {
this.queryParams = queryParams;
}
public void addQueryParams(String queryOnlyParam) {
this.queryParams.add(queryOnlyParam) ;
}
public static String normalize(String server) {
if (server.endsWith("/"))
server = server.substring(0, server.length() - 1);
return server;
}
/**
* Tries to query a live server from the list provided in Req. Servers in the dead pool are skipped.
* If a request fails due to an IOException, the server is moved to the dead pool for a certain period of
* time, or until a test request on that server succeeds.
*
* Servers are queried in the exact order given (except servers currently in the dead pool are skipped).
* If no live servers from the provided list remain to be tried, a number of previously skipped dead servers will be tried.
* Req.getNumDeadServersToTry() controls how many dead servers will be tried.
*
* If no live servers are found a SolrServerException is thrown.
*
* @param req contains both the request as well as the list of servers to query
*
* @return the result of the request
*
* @throws IOException If there is a low-level I/O error.
*/
public Rsp request(Req req) throws SolrServerException, IOException {
Rsp rsp = new Rsp();
Exception ex = null;
boolean isNonRetryable = req.request instanceof IsUpdateRequest || ADMIN_PATHS.contains(req.request.getPath());
List<ServerWrapper> skipped = null;
final Integer numServersToTry = req.getNumServersToTry();
int numServersTried = 0;
boolean timeAllowedExceeded = false;
long timeAllowedNano = getTimeAllowedInNanos(req.getRequest());
long timeOutTime = System.nanoTime() + timeAllowedNano;
for (String serverStr : req.getServers()) {
if (timeAllowedExceeded = isTimeExceeded(timeAllowedNano, timeOutTime)) {
break;
}
serverStr = normalize(serverStr);
// if the server is currently a zombie, just skip to the next one
ServerWrapper wrapper = zombieServers.get(serverStr);
if (wrapper != null) {
// System.out.println("ZOMBIE SERVER QUERIED: " + serverStr);
final int numDeadServersToTry = req.getNumDeadServersToTry();
if (numDeadServersToTry > 0) {
if (skipped == null) {
skipped = new ArrayList<>(numDeadServersToTry);
skipped.add(wrapper);
}
else if (skipped.size() < numDeadServersToTry) {
skipped.add(wrapper);
}
}
continue;
}
try {
MDC.put("LBSolrClient.url", serverStr);
if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
break;
}
++numServersTried;
ex = doRequest(serverStr, req, rsp, isNonRetryable, false);
if (ex == null) {
return rsp; // SUCCESS
}
} finally {
MDC.remove("LBSolrClient.url");
}
}
// try the servers we previously skipped
if (skipped != null) {
for (ServerWrapper wrapper : skipped) {
if (timeAllowedExceeded = isTimeExceeded(timeAllowedNano, timeOutTime)) {
break;
}
if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
break;
}
try {
MDC.put("LBSolrClient.url", wrapper.getBaseUrl());
++numServersTried;
ex = doRequest(wrapper.baseUrl, req, rsp, isNonRetryable, true);
if (ex == null) {
return rsp; // SUCCESS
}
} finally {
MDC.remove("LBSolrClient.url");
}
}
}
final String solrServerExceptionMessage;
if (timeAllowedExceeded) {
solrServerExceptionMessage = "Time allowed to handle this request exceeded";
} else {
if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
solrServerExceptionMessage = "No live SolrServers available to handle this request:"
+ " numServersTried="+numServersTried
+ " numServersToTry="+numServersToTry.intValue();
} else {
solrServerExceptionMessage = "No live SolrServers available to handle this request";
}
}
if (ex == null) {
throw new SolrServerException(solrServerExceptionMessage);
} else {
throw new SolrServerException(solrServerExceptionMessage+":" + zombieServers.keySet(), ex);
}
}
/**
* @return time allowed in nanos, returns -1 if no time_allowed is specified.
*/
private long getTimeAllowedInNanos(final SolrRequest req) {
SolrParams reqParams = req.getParams();
return reqParams == null ? -1 :
TimeUnit.NANOSECONDS.convert(reqParams.getInt(CommonParams.TIME_ALLOWED, -1), TimeUnit.MILLISECONDS);
}
private boolean isTimeExceeded(long timeAllowedNano, long timeOutTime) {
return timeAllowedNano > 0 && System.nanoTime() > timeOutTime;
}
protected Exception doRequest(String baseUrl, Req req, Rsp rsp, boolean isNonRetryable,
boolean isZombie) throws SolrServerException, IOException {
Exception ex = null;
try {
rsp.server = baseUrl;
req.getRequest().setBasePath(baseUrl);
rsp.rsp = getClient(baseUrl).request(req.getRequest(), (String) null);
if (isZombie) {
zombieServers.remove(baseUrl);
}
} catch (HttpSolrClient.RemoteExecutionException e){
throw e;
} catch(SolrException e) {
// we retry on 404 or 403 or 503 or 500
// unless it's an update - then we only retry on connect exception
if (!isNonRetryable && RETRY_CODES.contains(e.code())) {
ex = (!isZombie) ? addZombie(baseUrl, e) : e;
} else {
// Server is alive but the request was likely malformed or invalid
if (isZombie) {
zombieServers.remove(baseUrl);
}
throw e;
}
} catch (SocketException e) {
if (!isNonRetryable || e instanceof ConnectException) {
ex = (!isZombie) ? addZombie(baseUrl, e) : e;
} else {
throw e;
}
} catch (SocketTimeoutException e) {
if (!isNonRetryable) {
ex = (!isZombie) ? addZombie(baseUrl, e) : e;
} else {
throw e;
}
} catch (SolrServerException e) {
Throwable rootCause = e.getRootCause();
if (!isNonRetryable && rootCause instanceof IOException) {
ex = (!isZombie) ? addZombie(baseUrl, e) : e;
} else if (isNonRetryable && rootCause instanceof ConnectException) {
ex = (!isZombie) ? addZombie(baseUrl, e) : e;
} else {
throw e;
}
} catch (Exception e) {
throw new SolrServerException(e);
}
return ex;
}
protected abstract SolrClient getClient(String baseUrl);
private Exception addZombie(String serverStr, Exception e) {
ServerWrapper wrapper = createServerWrapper(serverStr);
wrapper.standard = false;
zombieServers.put(serverStr, wrapper);
startAliveCheckExecutor();
return e;
}
/**
* LBHttpSolrServer keeps pinging the dead servers at fixed interval to find if it is alive. Use this to set that
* interval
*
* @param interval time in milliseconds
*/
public void setAliveCheckInterval(int interval) {
if (interval <= 0) {
throw new IllegalArgumentException("Alive check interval must be " +
"positive, specified value = " + interval);
}
this.interval = interval;
}
private void startAliveCheckExecutor() {
// double-checked locking, but it's OK because we don't *do* anything with aliveCheckExecutor
// if it's not null.
if (aliveCheckExecutor == null) {
synchronized (this) {
if (aliveCheckExecutor == null) {
aliveCheckExecutor = Executors.newSingleThreadScheduledExecutor(
new SolrjNamedThreadFactory("aliveCheckExecutor"));
aliveCheckExecutor.scheduleAtFixedRate(
getAliveCheckRunner(new WeakReference<>(this)),
this.interval, this.interval, TimeUnit.MILLISECONDS);
}
}
}
}
private static Runnable getAliveCheckRunner(final WeakReference<LBSolrClient> lbRef) {
return () -> {
LBSolrClient lb = lbRef.get();
if (lb != null && lb.zombieServers != null) {
for (Object zombieServer : lb.zombieServers.values()) {
lb.checkAZombieServer((ServerWrapper)zombieServer);
}
}
};
}
public ResponseParser getParser() {
return parser;
}
/**
* Changes the {@link ResponseParser} that will be used for the internal
* SolrServer objects.
*
* @param parser Default Response Parser chosen to parse the response if the parser
* were not specified as part of the request.
* @see org.apache.solr.client.solrj.SolrRequest#getResponseParser()
*/
public void setParser(ResponseParser parser) {
this.parser = parser;
}
/**
* Changes the {@link RequestWriter} that will be used for the internal
* SolrServer objects.
*
* @param requestWriter Default RequestWriter, used to encode requests sent to the server.
*/
public void setRequestWriter(RequestWriter requestWriter) {
this.requestWriter = requestWriter;
}
public RequestWriter getRequestWriter() {
return requestWriter;
}
private void checkAZombieServer(ServerWrapper zombieServer) {
try {
QueryRequest queryRequest = new QueryRequest(solrQuery);
queryRequest.setBasePath(zombieServer.baseUrl);
QueryResponse resp = queryRequest.process(getClient(zombieServer.getBaseUrl()));
if (resp.getStatus() == 0) {
// server has come back up.
// make sure to remove from zombies before adding to alive to avoid a race condition
// where another thread could mark it down, move it back to zombie, and then we delete
// from zombie and lose it forever.
ServerWrapper wrapper = zombieServers.remove(zombieServer.getBaseUrl());
if (wrapper != null) {
wrapper.failedPings = 0;
if (wrapper.standard) {
addToAlive(wrapper);
}
} else {
// something else already moved the server from zombie to alive
}
}
} catch (Exception e) {
//Expected. The server is still down.
zombieServer.failedPings++;
// If the server doesn't belong in the standard set belonging to this load balancer
// then simply drop it after a certain number of failed pings.
if (!zombieServer.standard && zombieServer.failedPings >= NONSTANDARD_PING_LIMIT) {
zombieServers.remove(zombieServer.getBaseUrl());
}
}
}
private ServerWrapper removeFromAlive(String key) {
synchronized (aliveServers) {
ServerWrapper wrapper = aliveServers.remove(key);
if (wrapper != null)
updateAliveList();
return wrapper;
}
}
private void addToAlive(ServerWrapper wrapper) {
synchronized (aliveServers) {
ServerWrapper prev = aliveServers.put(wrapper.getBaseUrl(), wrapper);
// TODO: warn if there was a previous entry?
updateAliveList();
}
}
public void addSolrServer(String server) throws MalformedURLException {
addToAlive(createServerWrapper(server));
}
public String removeSolrServer(String server) {
try {
server = new URL(server).toExternalForm();
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
if (server.endsWith("/")) {
server = server.substring(0, server.length() - 1);
}
// there is a small race condition here - if the server is in the process of being moved between
// lists, we could fail to remove it.
removeFromAlive(server);
zombieServers.remove(server);
return null;
}
/**
* Tries to query a live server. A SolrServerException is thrown if all servers are dead.
* If the request failed due to IOException then the live server is moved to dead pool and the request is
* retried on another live server. After live servers are exhausted, any servers previously marked as dead
* will be tried before failing the request.
*
* @param request the SolrRequest.
*
* @return response
*
* @throws IOException If there is a low-level I/O error.
*/
@Override
public NamedList<Object> request(final SolrRequest request, String collection)
throws SolrServerException, IOException {
return request(request, collection, null);
}
public NamedList<Object> request(final SolrRequest request, String collection,
final Integer numServersToTry) throws SolrServerException, IOException {
Exception ex = null;
ServerWrapper[] serverList = aliveServerList;
final int maxTries = (numServersToTry == null ? serverList.length : numServersToTry.intValue());
int numServersTried = 0;
Map<String,ServerWrapper> justFailed = null;
boolean timeAllowedExceeded = false;
long timeAllowedNano = getTimeAllowedInNanos(request);
long timeOutTime = System.nanoTime() + timeAllowedNano;
for (int attempts=0; attempts<maxTries; attempts++) {
if (timeAllowedExceeded = isTimeExceeded(timeAllowedNano, timeOutTime)) {
break;
}
ServerWrapper wrapper = pickServer(serverList, request);
try {
++numServersTried;
request.setBasePath(wrapper.baseUrl);
return getClient(wrapper.getBaseUrl()).request(request, collection);
} catch (SolrException e) {
// Server is alive but the request was malformed or invalid
throw e;
} catch (SolrServerException e) {
if (e.getRootCause() instanceof IOException) {
ex = e;
moveAliveToDead(wrapper);
if (justFailed == null) justFailed = new HashMap<>();
justFailed.put(wrapper.getBaseUrl(), wrapper);
} else {
throw e;
}
} catch (Exception e) {
throw new SolrServerException(e);
}
}
// try other standard servers that we didn't try just now
for (ServerWrapper wrapper : zombieServers.values()) {
if (timeAllowedExceeded = isTimeExceeded(timeAllowedNano, timeOutTime)) {
break;
}
if (wrapper.standard==false || justFailed!=null && justFailed.containsKey(wrapper.getBaseUrl())) continue;
try {
++numServersTried;
request.setBasePath(wrapper.baseUrl);
NamedList<Object> rsp = getClient(wrapper.baseUrl).request(request, collection);
// remove from zombie list *before* adding to alive to avoid a race that could lose a server
zombieServers.remove(wrapper.getBaseUrl());
addToAlive(wrapper);
return rsp;
} catch (SolrException e) {
// Server is alive but the request was malformed or invalid
throw e;
} catch (SolrServerException e) {
if (e.getRootCause() instanceof IOException) {
ex = e;
// still dead
} else {
throw e;
}
} catch (Exception e) {
throw new SolrServerException(e);
}
}
final String solrServerExceptionMessage;
if (timeAllowedExceeded) {
solrServerExceptionMessage = "Time allowed to handle this request exceeded";
} else {
if (numServersToTry != null && numServersTried > numServersToTry.intValue()) {
solrServerExceptionMessage = "No live SolrServers available to handle this request:"
+ " numServersTried="+numServersTried
+ " numServersToTry="+numServersToTry.intValue();
} else {
solrServerExceptionMessage = "No live SolrServers available to handle this request";
}
}
if (ex == null) {
throw new SolrServerException(solrServerExceptionMessage);
} else {
throw new SolrServerException(solrServerExceptionMessage, ex);
}
}
/**
* Pick a server from list to execute request.
* By default servers are picked in round-robin manner,
* custom classes can override this method for more advance logic
* @param aliveServerList list of currently alive servers
* @param request the request will be sent to the picked server
* @return the picked server
*/
protected ServerWrapper pickServer(ServerWrapper[] aliveServerList, SolrRequest request) {
int count = counter.incrementAndGet() & Integer.MAX_VALUE;
return aliveServerList[count % aliveServerList.length];
}
private void moveAliveToDead(ServerWrapper wrapper) {
wrapper = removeFromAlive(wrapper.getBaseUrl());
if (wrapper == null)
return; // another thread already detected the failure and removed it
zombieServers.put(wrapper.getBaseUrl(), wrapper);
startAliveCheckExecutor();
}
@Override
public void close() {
synchronized (this) {
if (aliveCheckExecutor != null) {
aliveCheckExecutor.shutdownNow();
ExecutorUtil.shutdownAndAwaitTermination(aliveCheckExecutor);
}
}
}
}

46
solr/solrj/src/java/org/apache/solr/client/solrj/impl/PreemptiveBasicAuthClientBuilderFactory.java
View File

@ -31,10 +31,13 @@ import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder.CredentialsProviderProvider;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.StrUtils;
import org.eclipse.jetty.client.HttpAuthenticationStore;
import org.eclipse.jetty.client.ProxyAuthenticationProtocolHandler;
import org.apache.solr.client.solrj.util.SolrBasicAuthentication;
import org.eclipse.jetty.client.WWWAuthenticationProtocolHandler;
/**
* HttpClientConfigurer implementation providing support for preemptive Http Basic authentication
@ -104,26 +107,39 @@ public class PreemptiveBasicAuthClientBuilderFactory implements HttpClientBuilde
}
@Override
public SolrHttpClientBuilder getHttpClientBuilder(Optional<SolrHttpClientBuilder> builder) {
return builder.isPresent() ?
initHttpClientBuilder(builder.get())
: initHttpClientBuilder(SolrHttpClientBuilder.create());
}
private SolrHttpClientBuilder initHttpClientBuilder(SolrHttpClientBuilder builder) {
public void setup(Http2SolrClient client) {
final String basicAuthUser = defaultParams.get(HttpClientUtil.PROP_BASIC_AUTH_USER);
final String basicAuthPass = defaultParams.get(HttpClientUtil.PROP_BASIC_AUTH_PASS);
if(basicAuthUser == null || basicAuthPass == null) {
throw new IllegalArgumentException("username & password must be specified with " + getClass().getName());
}
builder.setDefaultCredentialsProvider(new CredentialsProviderProvider() {
@Override
public CredentialsProvider getCredentialsProvider() {
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(basicAuthUser, basicAuthPass));
return credsProvider;
}
HttpAuthenticationStore authenticationStore = new HttpAuthenticationStore();
authenticationStore.addAuthentication(new SolrBasicAuthentication(basicAuthUser, basicAuthPass));
client.getHttpClient().setAuthenticationStore(authenticationStore);
client.getProtocolHandlers().put(new WWWAuthenticationProtocolHandler(client.getHttpClient()));
client.getProtocolHandlers().put(new ProxyAuthenticationProtocolHandler(client.getHttpClient()));
}
@Override
public SolrHttpClientBuilder getHttpClientBuilder(Optional<SolrHttpClientBuilder> optionalBuilder) {
final String basicAuthUser = defaultParams.get(HttpClientUtil.PROP_BASIC_AUTH_USER);
final String basicAuthPass = defaultParams.get(HttpClientUtil.PROP_BASIC_AUTH_PASS);
if(basicAuthUser == null || basicAuthPass == null) {
throw new IllegalArgumentException("username & password must be specified with " + getClass().getName());
}
SolrHttpClientBuilder builder = optionalBuilder.isPresent() ?
initHttpClientBuilder(optionalBuilder.get(), basicAuthUser, basicAuthPass)
: initHttpClientBuilder(SolrHttpClientBuilder.create(), basicAuthUser, basicAuthPass);
return builder;
}
private SolrHttpClientBuilder initHttpClientBuilder(SolrHttpClientBuilder builder, String basicAuthUser, String basicAuthPass) {
builder.setDefaultCredentialsProvider(() -> {
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(basicAuthUser, basicAuthPass));
return credsProvider;
});
HttpClientUtil.addRequestInterceptor(requestInterceptor);

41
solr/solrj/src/java/org/apache/solr/client/solrj/util/Constants.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.util;
import java.util.StringTokenizer;
// Clone of org.apache.lucene.util.Constants, so SolrJ can use it
public class Constants {
public static final String JVM_SPEC_VERSION = System.getProperty("java.specification.version");
private static final int JVM_MAJOR_VERSION;
private static final int JVM_MINOR_VERSION;
static {
final StringTokenizer st = new StringTokenizer(JVM_SPEC_VERSION, ".");
JVM_MAJOR_VERSION = Integer.parseInt(st.nextToken());
if (st.hasMoreTokens()) {
JVM_MINOR_VERSION = Integer.parseInt(st.nextToken());
} else {
JVM_MINOR_VERSION = 0;
}
}
public static final boolean JRE_IS_MINIMUM_JAVA9 = JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 9);
public static final boolean JRE_IS_MINIMUM_JAVA11 = JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 11);
}

61
solr/solrj/src/java/org/apache/solr/client/solrj/util/SolrBasicAuthentication.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.util;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.Locale;
import org.eclipse.jetty.client.api.Authentication;
import org.eclipse.jetty.client.api.ContentResponse;
import org.eclipse.jetty.client.api.Request;
import org.eclipse.jetty.util.Attributes;
import org.eclipse.jetty.util.B64Code;
/**
* BasicAuthentication that does not care about uri and realm
*/
public class SolrBasicAuthentication implements Authentication {
private final String value;
public SolrBasicAuthentication(String user, String password) {
this.value = "Basic " + B64Code.encode(user + ":" + password, StandardCharsets.ISO_8859_1);
}
@Override
public boolean matches(String type, URI uri, String realm) {
return true;
}
@Override
public Result authenticate(Request request, ContentResponse response, HeaderInfo headerInfo, Attributes context) {
return new Result() {
@Override
public URI getURI() {
// cache result by host and port
return URI.create(String.format(Locale.ROOT, "%s://%s:%d", request.getScheme(), request.getHost(), request.getPort()));
}
@Override
public void apply(Request request) {
request.header(headerInfo.getHeader(), value);
}
};
}
}

40
solr/solrj/src/test-files/log4j2.xml Normal file
View File

@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<Configuration>
<Appenders>
<Console name="STDERR" target="SYSTEM_ERR">
<PatternLayout>
<Pattern>
%-4r %-5p (%t) [%X{node_name} %X{collection} %X{shard} %X{replica} %X{core}] %c{1.} %m%n
</Pattern>
</PatternLayout>
</Console>
</Appenders>
<Loggers>
<Logger name="org.apache.zookeeper" level="WARN"/>
<Logger name="org.apache.hadoop" level="WARN"/>
<Logger name="org.apache.directory" level="WARN"/>
<Logger name="org.apache.solr.hadoop" level="INFO"/>
<Logger name="org.eclipse.jetty" level="INFO"/>
<Root level="INFO">
<AppenderRef ref="STDERR"/>
</Root>
</Loggers>
</Configuration>

58
solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleBinaryHttp2Test.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.junit.BeforeClass;
/**
* A subclass of SolrExampleTests that explicitly uses the binary
* codec for communication.
*/
@SolrTestCaseJ4.SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
public class SolrExampleBinaryHttp2Test extends SolrExampleTests {
@BeforeClass
public static void beforeTest() throws Exception {
createAndStartJetty(legacyExampleCollection1SolrHome());
}
@Override
public SolrClient createNewSolrClient()
{
try {
// setup the server...
String url = jetty.getBaseUrl().toString() + "/collection1";
Http2SolrClient client = new Http2SolrClient.Builder(url)
.connectionTimeout(DEFAULT_CONNECTION_TIMEOUT)
.build();
// where the magic happens
client.setParser(new BinaryResponseParser());
client.setRequestWriter(new BinaryRequestWriter());
return client;
}
catch( Exception ex ) {
throw new RuntimeException( ex );
}
}
}

8
solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
View File

@ -36,6 +36,7 @@ import junit.framework.Assert;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.client.solrj.embedded.SolrExampleStreamingHttp2Test;
import org.apache.solr.client.solrj.embedded.SolrExampleStreamingTest.ErrorTrackingConcurrentUpdateSolrClient;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
@ -1700,11 +1701,16 @@ abstract public class SolrExampleTests extends SolrExampleTestsBase
client.add(doc);
if(client instanceof HttpSolrClient) { //XXX concurrent client reports exceptions differently
fail("Operation should throw an exception!");
} else {
} else if (client instanceof ErrorTrackingConcurrentUpdateSolrClient) {
client.commit(); //just to be sure the client has sent the doc
ErrorTrackingConcurrentUpdateSolrClient concurrentClient = (ErrorTrackingConcurrentUpdateSolrClient) client;
assertNotNull("ConcurrentUpdateSolrClient did not report an error", concurrentClient.lastError);
assertTrue("ConcurrentUpdateSolrClient did not report an error", concurrentClient.lastError.getMessage().contains("Conflict"));
} else if (client instanceof SolrExampleStreamingHttp2Test.ErrorTrackingConcurrentUpdateSolrClient) {
client.commit(); //just to be sure the client has sent the doc
SolrExampleStreamingHttp2Test.ErrorTrackingConcurrentUpdateSolrClient concurrentClient = (SolrExampleStreamingHttp2Test.ErrorTrackingConcurrentUpdateSolrClient) client;
assertNotNull("ConcurrentUpdateSolrClient did not report an error", concurrentClient.lastError);
assertTrue("ConcurrentUpdateSolrClient did not report an error", concurrentClient.lastError.getMessage().contains("conflict"));
}
} catch (SolrException se) {
assertTrue("No identifiable error message", se.getMessage().contains("version conflict for unique"));

335
solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttp2SolrClient.java Normal file
View File

@ -0,0 +1,335 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj;
import java.io.File;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.LBHttp2SolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.SolrResponseBase;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.util.TimeOut;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Test for LBHttp2SolrClient
*
* @since solr 1.4
*/
@Slow
@ThreadLeakFilters(defaultFilters = true, filters = {
SolrIgnoredThreadsFilter.class,
QuickPatchThreadsFilter.class
})
public class TestLBHttp2SolrClient extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
SolrInstance[] solr = new SolrInstance[3];
Http2SolrClient httpClient;
// TODO: fix this test to not require FSDirectory
static String savedFactory;
@BeforeClass
public static void beforeClass() {
savedFactory = System.getProperty("solr.DirectoryFactory");
System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockFSDirectoryFactory");
System.setProperty("tests.shardhandler.randomSeed", Long.toString(random().nextLong()));
}
@AfterClass
public static void afterClass() {
if (savedFactory == null) {
System.clearProperty("solr.directoryFactory");
} else {
System.setProperty("solr.directoryFactory", savedFactory);
}
System.clearProperty("tests.shardhandler.randomSeed");
}
@Override
public void setUp() throws Exception {
super.setUp();
httpClient = new Http2SolrClient.Builder().connectionTimeout(1000).idleTimeout(2000).build();
for (int i = 0; i < solr.length; i++) {
solr[i] = new SolrInstance("solr/collection1" + i, createTempDir("instance-" + i).toFile(), 0);
solr[i].setUp();
solr[i].startJetty();
addDocs(solr[i]);
}
}
private void addDocs(SolrInstance solrInstance) throws IOException, SolrServerException {
List<SolrInputDocument> docs = new ArrayList<>();
for (int i = 0; i < 10; i++) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", i);
doc.addField("name", solrInstance.name);
docs.add(doc);
}
SolrResponseBase resp;
try (HttpSolrClient client = getHttpSolrClient(solrInstance.getUrl())) {
resp = client.add(docs);
assertEquals(0, resp.getStatus());
resp = client.commit();
assertEquals(0, resp.getStatus());
}
}
@Override
public void tearDown() throws Exception {
for (SolrInstance aSolr : solr) {
if (aSolr != null) {
aSolr.tearDown();
}
}
httpClient.close();
super.tearDown();
}
public void testSimple() throws Exception {
String[] s = new String[solr.length];
for (int i = 0; i < solr.length; i++) {
s[i] = solr[i].getUrl();
}
try (LBHttp2SolrClient client = getLBHttp2SolrClient(httpClient, s)) {
client.setAliveCheckInterval(500);
SolrQuery solrQuery = new SolrQuery("*:*");
Set<String> names = new HashSet<>();
QueryResponse resp = null;
for (String ignored : s) {
resp = client.query(solrQuery);
assertEquals(10, resp.getResults().getNumFound());
names.add(resp.getResults().get(0).getFieldValue("name").toString());
}
assertEquals(3, names.size());
// Kill a server and test again
solr[1].jetty.stop();
solr[1].jetty = null;
names.clear();
for (String ignored : s) {
resp = client.query(solrQuery);
assertEquals(10, resp.getResults().getNumFound());
names.add(resp.getResults().get(0).getFieldValue("name").toString());
}
assertEquals(2, names.size());
assertFalse(names.contains("solr1"));
// Start the killed server once again
solr[1].startJetty();
// Wait for the alive check to complete
Thread.sleep(1200);
names.clear();
for (String ignored : s) {
resp = client.query(solrQuery);
assertEquals(10, resp.getResults().getNumFound());
names.add(resp.getResults().get(0).getFieldValue("name").toString());
}
assertEquals(3, names.size());
}
}
private LBHttp2SolrClient getLBHttp2SolrClient(Http2SolrClient httpClient, String... s) {
return new LBHttp2SolrClient(httpClient, s);
}
public void testTwoServers() throws Exception {
try (LBHttp2SolrClient client = getLBHttp2SolrClient(httpClient, solr[0].getUrl(), solr[1].getUrl())) {
client.setAliveCheckInterval(500);
SolrQuery solrQuery = new SolrQuery("*:*");
QueryResponse resp = null;
solr[0].jetty.stop();
solr[0].jetty = null;
resp = client.query(solrQuery);
String name = resp.getResults().get(0).getFieldValue("name").toString();
Assert.assertEquals("solr/collection11", name);
resp = client.query(solrQuery);
name = resp.getResults().get(0).getFieldValue("name").toString();
Assert.assertEquals("solr/collection11", name);
solr[1].jetty.stop();
solr[1].jetty = null;
solr[0].startJetty();
Thread.sleep(1200);
try {
resp = client.query(solrQuery);
} catch(SolrServerException e) {
// try again after a pause in case the error is lack of time to start server
Thread.sleep(3000);
resp = client.query(solrQuery);
}
name = resp.getResults().get(0).getFieldValue("name").toString();
Assert.assertEquals("solr/collection10", name);
}
}
public void testReliability() throws Exception {
String[] s = new String[solr.length];
for (int i = 0; i < solr.length; i++) {
s[i] = solr[i].getUrl();
}
try(LBHttp2SolrClient client = getLBHttp2SolrClient(httpClient, s)) {
client.setAliveCheckInterval(500);
// Kill a server and test again
solr[1].jetty.stop();
solr[1].jetty = null;
// query the servers
for (String value : s)
client.query(new SolrQuery("*:*"));
// Start the killed server once again
solr[1].startJetty();
// Wait for the alive check to complete
waitForServer(30, client, 3, solr[1].name);
}
}
// wait maximum ms for serverName to come back up
private void waitForServer(int maxSeconds, LBHttp2SolrClient client, int nServers, String serverName) throws Exception {
final TimeOut timeout = new TimeOut(maxSeconds, TimeUnit.SECONDS, TimeSource.NANO_TIME);
while (! timeout.hasTimedOut()) {
QueryResponse resp;
try {
resp = client.query(new SolrQuery("*:*"));
} catch (Exception e) {
log.warn("", e);
continue;
}
String name = resp.getResults().get(0).getFieldValue("name").toString();
if (name.equals(serverName))
return;
Thread.sleep(500);
}
}
private static class SolrInstance {
String name;
File homeDir;
File dataDir;
File confDir;
int port;
JettySolrRunner jetty;
public SolrInstance(String name, File homeDir, int port) {
this.name = name;
this.homeDir = homeDir;
this.port = port;
dataDir = new File(homeDir + "/collection1", "data");
confDir = new File(homeDir + "/collection1", "conf");
}
public String getHomeDir() {
return homeDir.toString();
}
public String getUrl() {
return buildUrl(port, "/solr/collection1");
}
public String getSchemaFile() {
return "solrj/solr/collection1/conf/schema-replication1.xml";
}
public String getConfDir() {
return confDir.toString();
}
public String getDataDir() {
return dataDir.toString();
}
public String getSolrConfigFile() {
return "solrj/solr/collection1/conf/solrconfig-slave1.xml";
}
public String getSolrXmlFile() {
return "solrj/solr/solr.xml";
}
public void setUp() throws Exception {
homeDir.mkdirs();
dataDir.mkdirs();
confDir.mkdirs();
FileUtils.copyFile(SolrTestCaseJ4.getFile(getSolrXmlFile()), new File(homeDir, "solr.xml"));
File f = new File(confDir, "solrconfig.xml");
FileUtils.copyFile(SolrTestCaseJ4.getFile(getSolrConfigFile()), f);
f = new File(confDir, "schema.xml");
FileUtils.copyFile(SolrTestCaseJ4.getFile(getSchemaFile()), f);
Files.createFile(homeDir.toPath().resolve("collection1/core.properties"));
}
public void tearDown() throws Exception {
if (jetty != null) jetty.stop();
IOUtils.rm(homeDir.toPath());
}
public void startJetty() throws Exception {
Properties props = new Properties();
props.setProperty("solrconfig", "bad_solrconfig.xml");
props.setProperty("solr.data.dir", getDataDir());
JettyConfig jettyConfig = JettyConfig.builder(buildJettyConfig("/solr")).setPort(port).build();
jetty = new JettySolrRunner(getHomeDir(), props, jettyConfig);
jetty.start();
int newPort = jetty.getLocalPort();
if (port != 0 && newPort != port) {
fail("TESTING FAILURE: could not grab requested port.");
}
this.port = newPort;
// System.out.println("waiting.........");
// Thread.sleep(5000);
}
}
}

103
solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingBinaryHttp2Test.java Normal file
View File

@ -0,0 +1,103 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.embedded;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.StreamingResponseCallback;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.junit.Test;
@LuceneTestCase.Slow
@SolrTestCaseJ4.SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
public class SolrExampleStreamingBinaryHttp2Test extends SolrExampleStreamingHttp2Test {
@Override
public SolrClient createNewSolrClient() {
// setup the server...
String url = jetty.getBaseUrl().toString() + "/collection1";
// smaller queue size hits locks more often
Http2SolrClient solrClient = new Http2SolrClient.Builder()
.build();
solrClient.setParser(new BinaryResponseParser());
solrClient.setRequestWriter(new BinaryRequestWriter());
ConcurrentUpdateHttp2SolrClient concurrentClient = new ErrorTrackingConcurrentUpdateSolrClient.Builder(url, solrClient)
.withQueueSize(2)
.withThreadCount(5)
.build();
return concurrentClient;
}
@Test
public void testQueryAndStreamResponse() throws Exception {
// index a simple document with one child
SolrClient client = getSolrClient();
client.deleteByQuery("*:*");
SolrInputDocument child = new SolrInputDocument();
child.addField("id", "child");
child.addField("type_s", "child");
child.addField("text_s", "text");
SolrInputDocument parent = new SolrInputDocument();
parent.addField("id", "parent");
parent.addField("type_s", "parent");
parent.addChildDocument(child);
client.add(parent);
client.commit();
// create a query with child doc transformer
SolrQuery query = new SolrQuery("{!parent which='type_s:parent'}text_s:text");
query.addField("*,[child parentFilter='type_s:parent']");
// test regular query
QueryResponse response = client.query(query);
assertEquals(1, response.getResults().size());
SolrDocument parentDoc = response.getResults().get(0);
assertEquals(1, parentDoc.getChildDocumentCount());
// test streaming
final List<SolrDocument> docs = new ArrayList<>();
client.queryAndStreamResponse(query, new StreamingResponseCallback() {
@Override
public void streamSolrDocument(SolrDocument doc) {
docs.add(doc);
}
@Override
public void streamDocListInfo(long numFound, long start, Float maxScore) {
}
});
assertEquals(1, docs.size());
parentDoc = docs.get(0);
assertEquals(1, parentDoc.getChildDocumentCount());
}
}

139
solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingHttp2Test.java Normal file
View File

@ -0,0 +1,139 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.embedded;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrExampleTests;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.XMLResponseParser;
import org.apache.solr.client.solrj.request.RequestWriter;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrInputDocument;
import org.junit.BeforeClass;
public class SolrExampleStreamingHttp2Test extends SolrExampleTests {
@BeforeClass
public static void beforeTest() throws Exception {
createAndStartJetty(legacyExampleCollection1SolrHome());
}
@Override
public SolrClient createNewSolrClient()
{
// setup the server...
String url = jetty.getBaseUrl().toString() + "/collection1";
// smaller queue size hits locks more often
Http2SolrClient solrClient = new Http2SolrClient.Builder()
.build();
solrClient.setParser(new XMLResponseParser());
solrClient.setRequestWriter(new RequestWriter());
ConcurrentUpdateHttp2SolrClient concurrentClient = new ErrorTrackingConcurrentUpdateSolrClient.Builder(url, solrClient)
.withQueueSize(2)
.withThreadCount(5)
.build();
return concurrentClient;
}
public void testWaitOptions() throws Exception {
// SOLR-3903
final List<Throwable> failures = new ArrayList<>();
final String serverUrl = jetty.getBaseUrl().toString() + "/collection1";
try (Http2SolrClient http2Client = new Http2SolrClient.Builder().build();
ConcurrentUpdateHttp2SolrClient concurrentClient = new FailureRecordingConcurrentUpdateSolrClient.Builder(serverUrl, http2Client)
.withQueueSize(2)
.withThreadCount(2)
.build()) {
int docId = 42;
for (UpdateRequest.ACTION action : EnumSet.allOf(UpdateRequest.ACTION.class)) {
for (boolean waitSearch : Arrays.asList(true, false)) {
for (boolean waitFlush : Arrays.asList(true, false)) {
UpdateRequest updateRequest = new UpdateRequest();
SolrInputDocument document = new SolrInputDocument();
document.addField("id", docId++);
updateRequest.add(document);
updateRequest.setAction(action, waitSearch, waitFlush);
concurrentClient.request(updateRequest);
}
}
}
concurrentClient.commit();
concurrentClient.blockUntilFinished();
}
if (0 != failures.size()) {
assertEquals(failures.size() + " Unexpected Exception, starting with...",
null, failures.get(0));
}
}
static class FailureRecordingConcurrentUpdateSolrClient extends ConcurrentUpdateHttp2SolrClient {
private final List<Throwable> failures = new ArrayList<>();
public FailureRecordingConcurrentUpdateSolrClient(Builder builder) {
super(builder);
}
@Override
public void handleError(Throwable ex) {
failures.add(ex);
}
static class Builder extends ConcurrentUpdateHttp2SolrClient.Builder {
public Builder(String baseSolrUrl, Http2SolrClient http2Client) {
super(baseSolrUrl, http2Client);
}
@Override
public FailureRecordingConcurrentUpdateSolrClient build() {
return new FailureRecordingConcurrentUpdateSolrClient(this);
}
}
}
public static class ErrorTrackingConcurrentUpdateSolrClient extends ConcurrentUpdateHttp2SolrClient {
public Throwable lastError = null;
public ErrorTrackingConcurrentUpdateSolrClient(Builder builder) {
super(builder);
}
@Override
public void handleError(Throwable ex) {
lastError = ex;
}
public static class Builder extends ConcurrentUpdateHttp2SolrClient.Builder {
public Builder(String baseSolrUrl, Http2SolrClient http2Client) {
super(baseSolrUrl, http2Client, true);
}
@Override
public ErrorTrackingConcurrentUpdateSolrClient build() {
return new ErrorTrackingConcurrentUpdateSolrClient(this);
}
}
}
}

45
solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleXMLHttp2Test.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.embedded;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrExampleTests;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.XMLResponseParser;
import org.apache.solr.client.solrj.request.RequestWriter;
import org.junit.BeforeClass;
public class SolrExampleXMLHttp2Test extends SolrExampleTests {
@BeforeClass
public static void beforeTest() throws Exception {
createAndStartJetty(legacyExampleCollection1SolrHome());
}
@Override
public SolrClient createNewSolrClient() {
try {
String url = jetty.getBaseUrl().toString() + "/collection1";
Http2SolrClient client = new Http2SolrClient.Builder(url).connectionTimeout(DEFAULT_CONNECTION_TIMEOUT).build();
client.setParser(new XMLResponseParser());
client.setRequestWriter(new RequestWriter());
return client;
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
}

2
solr/solrj/src/test/org/apache/solr/client/solrj/impl/BasicHttpSolrClientTest.java
View File

@ -200,7 +200,7 @@ public class BasicHttpSolrClientTest extends SolrJettyTestBase {
.withServlet(new ServletHolder(RedirectServlet.class), "/redirect/*")
.withServlet(new ServletHolder(SlowServlet.class), "/slow/*")
.withServlet(new ServletHolder(DebugServlet.class), "/debug/*")
.withSSLConfig(sslConfig)
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
createAndStartJetty(legacyExampleCollection1SolrHome(), jettyConfig);
}

100
solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClientBadInputTest.java Normal file
View File

@ -0,0 +1,100 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import java.util.ArrayList;
import java.util.List;
import com.google.common.collect.Lists;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.internal.matchers.StringContains.containsString;
public class ConcurrentUpdateHttp2SolrClientBadInputTest extends SolrJettyTestBase {
private static final List<String> NULL_STR_LIST = null;
private static final List<String> EMPTY_STR_LIST = new ArrayList<>();
private static final String ANY_COLLECTION = "ANY_COLLECTION";
private static final int ANY_COMMIT_WITHIN_TIME = -1;
private static final int ANY_QUEUE_SIZE = 1;
private static final int ANY_MAX_NUM_THREADS = 1;
@BeforeClass
public static void beforeTest() throws Exception {
JettyConfig jettyConfig = JettyConfig.builder()
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
createAndStartJetty(legacyExampleCollection1SolrHome(), jettyConfig);
}
@Test
public void testDeleteByIdReportsInvalidIdLists() throws Exception {
try (Http2SolrClient http2Client = new Http2SolrClient.Builder().build();
SolrClient client = new ConcurrentUpdateHttp2SolrClient.Builder(jetty.getBaseUrl().toString() + "/" + ANY_COLLECTION, http2Client)
.withQueueSize(ANY_QUEUE_SIZE)
.withThreadCount(ANY_MAX_NUM_THREADS)
.build()) {
assertExceptionThrownWithMessageContaining(IllegalArgumentException.class, Lists.newArrayList("ids", "null"), () -> {
client.deleteById(NULL_STR_LIST);
});
assertExceptionThrownWithMessageContaining(IllegalArgumentException.class, Lists.newArrayList("ids", "empty"), () -> {
client.deleteById(EMPTY_STR_LIST);
});
assertExceptionThrownWithMessageContaining(IllegalArgumentException.class, Lists.newArrayList("ids", "null"), () -> {
client.deleteById(NULL_STR_LIST, ANY_COMMIT_WITHIN_TIME);
});
assertExceptionThrownWithMessageContaining(IllegalArgumentException.class, Lists.newArrayList("ids", "empty"), () -> {
client.deleteById(EMPTY_STR_LIST, ANY_COMMIT_WITHIN_TIME);
});
}
try (Http2SolrClient http2Client = new Http2SolrClient.Builder().build();
SolrClient client = new ConcurrentUpdateHttp2SolrClient.Builder(jetty.getBaseUrl().toString() + "/" + ANY_COLLECTION, http2Client)
.withQueueSize(ANY_QUEUE_SIZE)
.withThreadCount(ANY_MAX_NUM_THREADS)
.build()) {
assertExceptionThrownWithMessageContaining(IllegalArgumentException.class, Lists.newArrayList("ids", "null"), () -> {
client.deleteById(ANY_COLLECTION, NULL_STR_LIST);
});
assertExceptionThrownWithMessageContaining(IllegalArgumentException.class, Lists.newArrayList("ids", "empty"), () -> {
client.deleteById(ANY_COLLECTION, EMPTY_STR_LIST);
});
assertExceptionThrownWithMessageContaining(IllegalArgumentException.class, Lists.newArrayList("ids", "null"), () -> {
client.deleteById(ANY_COLLECTION, NULL_STR_LIST, ANY_COMMIT_WITHIN_TIME);
});
assertExceptionThrownWithMessageContaining(IllegalArgumentException.class, Lists.newArrayList("ids", "empty"), () -> {
client.deleteById(ANY_COLLECTION, EMPTY_STR_LIST, ANY_COMMIT_WITHIN_TIME);
});
}
}
private void assertExceptionThrownWithMessageContaining(Class expectedType, List<String> expectedStrings, LuceneTestCase.ThrowingRunnable runnable) {
Throwable thrown = expectThrows(expectedType, runnable);
if (expectedStrings != null) {
for (String expectedString : expectedStrings) {
assertThat(thrown.getMessage(), containsString(expectedString));
}
}
}
}

95
solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClientMultiCollectionTest.java Normal file
View File

@ -0,0 +1,95 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import java.io.File;
import java.io.IOException;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.util.ExternalPaths;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
/**
* {@link ConcurrentUpdateSolrClient} reuses the same HTTP connection to send multiple requests. These tests ensure
* that this connection-reuse never results in documents being sent to the wrong collection. See SOLR-12803
*/
public class ConcurrentUpdateHttp2SolrClientMultiCollectionTest extends SolrCloudTestCase {
private static final String COLLECTION_ONE_NAME = "collection1";
private static final String COLLECTION_TWO_NAME = "collection2";
private String solrUrl;
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(1)
.addConfig("conf", new File(ExternalPaths.TECHPRODUCTS_CONFIGSET).toPath())
.configure();
}
@Before
public void createCollections() throws Exception {
solrUrl = cluster.getJettySolrRunner(0).getBaseUrl().toString();
CollectionAdminRequest.createCollection(COLLECTION_ONE_NAME, "conf", 1, 1).process(cluster.getSolrClient());
CollectionAdminRequest.createCollection(COLLECTION_TWO_NAME, "conf", 1, 1).process(cluster.getSolrClient());
}
@After
public void deleteCollections() throws Exception {
cluster.deleteAllCollections();
}
@Test
public void testEnsureDocumentsSentToCorrectCollection() throws Exception {
int numTotalDocs = 1000;
int numExpectedPerCollection = numTotalDocs / 2;
try (Http2SolrClient http2Client = new Http2SolrClient.Builder().build();
SolrClient client = new ConcurrentUpdateHttp2SolrClient.Builder(solrUrl, http2Client)
.withQueueSize(numTotalDocs).build()) {
splitDocumentsAcrossCollections(client, numTotalDocs);
assertEquals(numExpectedPerCollection, client.query(COLLECTION_ONE_NAME, new SolrQuery("*:*")).getResults().getNumFound());
assertEquals(numExpectedPerCollection, client.query(COLLECTION_TWO_NAME, new SolrQuery("*:*")).getResults().getNumFound());
}
}
private void splitDocumentsAcrossCollections(SolrClient client, int numTotalDocs) throws IOException, SolrServerException {
for (int docNum = 0; docNum < numTotalDocs; docNum++) {
final SolrInputDocument doc = new SolrInputDocument();
doc.setField("id", "value" + docNum);
if (docNum %2 == 0) {
client.add(COLLECTION_ONE_NAME, doc);
} else {
client.add(COLLECTION_TWO_NAME, doc);
}
}
client.commit(COLLECTION_ONE_NAME);
client.commit(COLLECTION_TWO_NAME);
}
}

233
solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClientTest.java Normal file
View File

@ -0,0 +1,233 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import java.io.IOException;
import java.io.InputStream;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.eclipse.jetty.client.api.Response;
import org.eclipse.jetty.servlet.ServletHolder;
import org.junit.BeforeClass;
import org.junit.Test;
public class ConcurrentUpdateHttp2SolrClientTest extends SolrJettyTestBase {
@BeforeClass
public static void beforeTest() throws Exception {
JettyConfig jettyConfig = JettyConfig.builder()
.withServlet(new ServletHolder(ConcurrentUpdateSolrClientTest.TestServlet.class), "/cuss/*")
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
createAndStartJetty(legacyExampleCollection1SolrHome(), jettyConfig);
}
@Test
public void testConcurrentUpdate() throws Exception {
ConcurrentUpdateSolrClientTest.TestServlet.clear();
String serverUrl = jetty.getBaseUrl().toString() + "/cuss/foo";
int cussThreadCount = 2;
int cussQueueSize = 100;
// for tracking callbacks from CUSS
final AtomicInteger successCounter = new AtomicInteger(0);
final AtomicInteger errorCounter = new AtomicInteger(0);
final StringBuilder errors = new StringBuilder();
try (Http2SolrClient http2Client = new Http2SolrClient.Builder().build();
ConcurrentUpdateHttp2SolrClient concurrentClient = new OutcomeCountingConcurrentUpdateSolrClient.Builder(serverUrl, http2Client, successCounter, errorCounter, errors)
.withQueueSize(cussQueueSize)
.withThreadCount(cussThreadCount)
.build()) {
concurrentClient.setPollQueueTime(0);
// ensure it doesn't block where there's nothing to do yet
concurrentClient.blockUntilFinished();
int poolSize = 5;
ExecutorService threadPool = ExecutorUtil.newMDCAwareFixedThreadPool(poolSize, new SolrjNamedThreadFactory("testCUSS"));
int numDocs = 100;
int numRunnables = 5;
for (int r=0; r < numRunnables; r++)
threadPool.execute(new ConcurrentUpdateSolrClientTest.SendDocsRunnable(String.valueOf(r), numDocs, concurrentClient));
// ensure all docs are sent
threadPool.awaitTermination(5, TimeUnit.SECONDS);
threadPool.shutdown();
// wait until all requests are processed by CUSS
concurrentClient.blockUntilFinished();
concurrentClient.shutdownNow();
assertEquals("post", ConcurrentUpdateSolrClientTest.TestServlet.lastMethod);
// expect all requests to be successful
int expectedSuccesses = ConcurrentUpdateSolrClientTest.TestServlet.numReqsRcvd.get();
assertTrue(expectedSuccesses > 0); // at least one request must have been sent
assertTrue("Expected no errors but got "+errorCounter.get()+
", due to: "+errors.toString(), errorCounter.get() == 0);
assertTrue("Expected "+expectedSuccesses+" successes, but got "+successCounter.get(),
successCounter.get() == expectedSuccesses);
int expectedDocs = numDocs * numRunnables;
assertTrue("Expected CUSS to send "+expectedDocs+" but got "+ ConcurrentUpdateSolrClientTest.TestServlet.numDocsRcvd.get(),
ConcurrentUpdateSolrClientTest.TestServlet.numDocsRcvd.get() == expectedDocs);
}
}
@Test
public void testCollectionParameters() throws IOException, SolrServerException {
int cussThreadCount = 2;
int cussQueueSize = 10;
try (Http2SolrClient http2Client = new Http2SolrClient.Builder().build();
ConcurrentUpdateHttp2SolrClient concurrentClient
= (new ConcurrentUpdateHttp2SolrClient.Builder(jetty.getBaseUrl().toString(), http2Client))
.withQueueSize(cussQueueSize)
.withThreadCount(cussThreadCount).build()) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", "collection");
concurrentClient.add("collection1", doc);
concurrentClient.commit("collection1");
assertEquals(1, concurrentClient.query("collection1", new SolrQuery("id:collection")).getResults().getNumFound());
}
try (Http2SolrClient http2Client = new Http2SolrClient.Builder().build();
ConcurrentUpdateHttp2SolrClient concurrentClient
= new ConcurrentUpdateHttp2SolrClient.Builder(jetty.getBaseUrl().toString() + "/collection1", http2Client)
.withQueueSize(cussQueueSize)
.withThreadCount(cussThreadCount).build()) {
assertEquals(1, concurrentClient.query(new SolrQuery("id:collection")).getResults().getNumFound());
}
}
@Test
public void testConcurrentCollectionUpdate() throws Exception {
int cussThreadCount = 2;
int cussQueueSize = 100;
int numDocs = 100;
int numRunnables = 5;
int expected = numDocs * numRunnables;
try (Http2SolrClient http2Client = new Http2SolrClient.Builder().build();
ConcurrentUpdateHttp2SolrClient concurrentClient
= new ConcurrentUpdateHttp2SolrClient.Builder(jetty.getBaseUrl().toString(), http2Client)
.withQueueSize(cussQueueSize)
.withThreadCount(cussThreadCount).build()) {
concurrentClient.setPollQueueTime(0);
// ensure it doesn't block where there's nothing to do yet
concurrentClient.blockUntilFinished();
// Delete all existing documents.
concurrentClient.deleteByQuery("collection1", "*:*");
int poolSize = 5;
ExecutorService threadPool = ExecutorUtil.newMDCAwareFixedThreadPool(poolSize, new SolrjNamedThreadFactory("testCUSS"));
for (int r=0; r < numRunnables; r++)
threadPool.execute(new ConcurrentUpdateSolrClientTest.SendDocsRunnable(String.valueOf(r), numDocs, concurrentClient, "collection1"));
// ensure all docs are sent
threadPool.awaitTermination(5, TimeUnit.SECONDS);
threadPool.shutdown();
concurrentClient.commit("collection1");
assertEquals(expected, concurrentClient.query("collection1", new SolrQuery("*:*")).getResults().getNumFound());
// wait until all requests are processed by CUSS
concurrentClient.blockUntilFinished();
concurrentClient.shutdownNow();
}
try (Http2SolrClient http2Client = new Http2SolrClient.Builder().build();
ConcurrentUpdateHttp2SolrClient concurrentClient
= new ConcurrentUpdateHttp2SolrClient.Builder(jetty.getBaseUrl().toString() + "/collection1", http2Client)
.withQueueSize(cussQueueSize)
.withThreadCount(cussThreadCount).build()) {
assertEquals(expected, concurrentClient.query(new SolrQuery("*:*")).getResults().getNumFound());
}
}
static class OutcomeCountingConcurrentUpdateSolrClient extends ConcurrentUpdateHttp2SolrClient {
private final AtomicInteger successCounter;
private final AtomicInteger failureCounter;
private final StringBuilder errors;
public OutcomeCountingConcurrentUpdateSolrClient(OutcomeCountingConcurrentUpdateSolrClient.Builder builder) {
super(builder);
this.successCounter = builder.successCounter;
this.failureCounter = builder.failureCounter;
this.errors = builder.errors;
}
@Override
public void handleError(Throwable ex) {
failureCounter.incrementAndGet();
errors.append(" "+ex);
}
@Override
public void onSuccess(Response resp, InputStream respBody) {
successCounter.incrementAndGet();
}
static class Builder extends ConcurrentUpdateHttp2SolrClient.Builder {
protected final AtomicInteger successCounter;
protected final AtomicInteger failureCounter;
protected final StringBuilder errors;
public Builder(String baseSolrUrl, Http2SolrClient http2Client, AtomicInteger successCounter, AtomicInteger failureCounter, StringBuilder errors) {
super(baseSolrUrl, http2Client);
this.successCounter = successCounter;
this.failureCounter = failureCounter;
this.errors = errors;
}
public OutcomeCountingConcurrentUpdateSolrClient build() {
return new OutcomeCountingConcurrentUpdateSolrClient(this);
}
}
}
}

2
solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBadInputTest.java
View File

@ -41,7 +41,7 @@ public class ConcurrentUpdateSolrClientBadInputTest extends SolrJettyTestBase {
@BeforeClass
public static void beforeTest() throws Exception {
JettyConfig jettyConfig = JettyConfig.builder()
.withSSLConfig(sslConfig)
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
createAndStartJetty(legacyExampleCollection1SolrHome(), jettyConfig);
}

9
solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientTest.java
View File

@ -18,6 +18,7 @@ package org.apache.solr.client.solrj.impl;
import org.apache.http.HttpResponse;
import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettyConfig;
@ -128,7 +129,7 @@ public class ConcurrentUpdateSolrClientTest extends SolrJettyTestBase {
public static void beforeTest() throws Exception {
JettyConfig jettyConfig = JettyConfig.builder()
.withServlet(new ServletHolder(TestServlet.class), "/cuss/*")
.withSSLConfig(sslConfig)
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
createAndStartJetty(legacyExampleCollection1SolrHome(), jettyConfig);
}
@ -273,14 +274,14 @@ public class ConcurrentUpdateSolrClientTest extends SolrJettyTestBase {
private String id;
private int numDocs;
private ConcurrentUpdateSolrClient cuss;
private SolrClient cuss;
private String collection;
SendDocsRunnable(String id, int numDocs, ConcurrentUpdateSolrClient cuss) {
SendDocsRunnable(String id, int numDocs, SolrClient cuss) {
this(id, numDocs, cuss, null);
}
SendDocsRunnable(String id, int numDocs, ConcurrentUpdateSolrClient cuss, String collection) {
SendDocsRunnable(String id, int numDocs, SolrClient cuss, String collection) {
this.id = id;
this.numDocs = numDocs;
this.cuss = cuss;

114
solr/solrj/src/test/org/apache/solr/client/solrj/impl/Http2SolrClientCompatibilityTest.java Normal file
View File

@ -0,0 +1,114 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import org.apache.http.ParseException;
import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.util.LogLevel;
import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
import org.eclipse.jetty.http2.client.http.HttpClientTransportOverHTTP2;
import org.eclipse.jetty.servlet.ServletHolder;
@LogLevel("org.eclipse.jetty.client=DEBUG;org.eclipse.jetty.util=DEBUG")
@SolrTestCaseJ4.SuppressSSL
public class Http2SolrClientCompatibilityTest extends SolrJettyTestBase {
public void testSystemPropertyFlag() {
System.setProperty("solr.http1", "true");
try (Http2SolrClient client = new Http2SolrClient.Builder()
.build()) {
assertTrue(client.getHttpClient().getTransport() instanceof HttpClientTransportOverHTTP);
}
System.clearProperty("solr.http1");
try (Http2SolrClient client = new Http2SolrClient.Builder()
.build()) {
assertTrue(client.getHttpClient().getTransport() instanceof HttpClientTransportOverHTTP2);
}
}
public void testConnectToOldNodesUsingHttp1() throws Exception {
JettyConfig jettyConfig = JettyConfig.builder()
.withServlet(new ServletHolder(Http2SolrClientTest.DebugServlet.class), "/debug/*")
.useOnlyHttp1(true)
.build();
createAndStartJetty(legacyExampleCollection1SolrHome(), jettyConfig);
try (Http2SolrClient client = new Http2SolrClient.Builder(jetty.getBaseUrl().toString() + "/debug/foo")
.useHttp1_1(true)
.build()) {
assertTrue(client.getHttpClient().getTransport() instanceof HttpClientTransportOverHTTP);
try {
client.query(new SolrQuery("*:*"), SolrRequest.METHOD.GET);
} catch (ParseException ignored) {}
} finally {
afterSolrJettyTestBase();
}
}
public void testConnectToNewNodesUsingHttp1() throws Exception {
JettyConfig jettyConfig = JettyConfig.builder()
.withServlet(new ServletHolder(Http2SolrClientTest.DebugServlet.class), "/debug/*")
.useOnlyHttp1(false)
.build();
createAndStartJetty(legacyExampleCollection1SolrHome(), jettyConfig);
try (Http2SolrClient client = new Http2SolrClient.Builder(jetty.getBaseUrl().toString() + "/debug/foo")
.useHttp1_1(true)
.build()) {
assertTrue(client.getHttpClient().getTransport() instanceof HttpClientTransportOverHTTP);
try {
client.query(new SolrQuery("*:*"), SolrRequest.METHOD.GET);
} catch (ParseException ignored) {}
} finally {
afterSolrJettyTestBase();
}
}
public void testConnectToOldNodesUsingHttp2() throws Exception {
// if this test some how failure, this mean that Jetty client now be able to switch between HTTP/1
// and HTTP/2.2 protocol dynamically therefore rolling updates will be easier we should then notify this to users
JettyConfig jettyConfig = JettyConfig.builder()
.withServlet(new ServletHolder(Http2SolrClientTest.DebugServlet.class), "/debug/*")
.useOnlyHttp1(true)
.build();
createAndStartJetty(legacyExampleCollection1SolrHome(), jettyConfig);
System.clearProperty("solr.http1");
try (Http2SolrClient client = new Http2SolrClient.Builder(jetty.getBaseUrl().toString() + "/debug/foo")
.build()) {
assertTrue(client.getHttpClient().getTransport() instanceof HttpClientTransportOverHTTP2);
try {
client.query(new SolrQuery("*:*"), SolrRequest.METHOD.GET);
fail("Jetty client with HTTP2 transport should not be able to connect to HTTP1 only nodes");
} catch (ParseException ignored) {
fail("Jetty client with HTTP2 transport should not be able to connect to HTTP1 only nodes");
} catch (SolrServerException e) {
// expected
}
} finally {
afterSolrJettyTestBase();
}
}
}

598
solr/solrj/src/test/org/apache/solr/client/solrj/impl/Http2SolrClientTest.java Normal file
View File

@ -0,0 +1,598 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.impl;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import org.apache.http.ParseException;
import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.request.RequestWriter;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.SuppressForbidden;
import org.eclipse.jetty.servlet.ServletHolder;
import org.junit.BeforeClass;
import org.junit.Test;
public class Http2SolrClientTest extends SolrJettyTestBase {
private static final String EXPECTED_USER_AGENT = "Solr[" + Http2SolrClient.class.getName() + "] 2.0";
public static class DebugServlet extends HttpServlet {
public static void clear() {
lastMethod = null;
headers = null;
parameters = null;
errorCode = null;
queryString = null;
cookies = null;
}
public static Integer errorCode = null;
public static String lastMethod = null;
public static HashMap<String,String> headers = null;
public static Map<String,String[]> parameters = null;
public static String queryString = null;
public static javax.servlet.http.Cookie[] cookies = null;
public static void setErrorCode(Integer code) {
errorCode = code;
}
@Override
protected void doDelete(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
lastMethod = "delete";
recordRequest(req, resp);
}
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
lastMethod = "get";
recordRequest(req, resp);
}
@Override
protected void doHead(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
lastMethod = "head";
recordRequest(req, resp);
}
private void setHeaders(HttpServletRequest req) {
Enumeration<String> headerNames = req.getHeaderNames();
headers = new HashMap<>();
while (headerNames.hasMoreElements()) {
final String name = headerNames.nextElement();
headers.put(name.toLowerCase(Locale.getDefault()), req.getHeader(name));
}
}
@SuppressForbidden(reason = "fake servlet only")
private void setParameters(HttpServletRequest req) {
parameters = req.getParameterMap();
}
private void setQueryString(HttpServletRequest req) {
queryString = req.getQueryString();
}
private void setCookies(HttpServletRequest req) {
javax.servlet.http.Cookie[] ck = req.getCookies();
cookies = req.getCookies();
}
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
lastMethod = "post";
recordRequest(req, resp);
}
@Override
protected void doPut(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
lastMethod = "put";
recordRequest(req, resp);
}
private void recordRequest(HttpServletRequest req, HttpServletResponse resp) {
setHeaders(req);
setParameters(req);
setQueryString(req);
setCookies(req);
if (null != errorCode) {
try {
resp.sendError(errorCode);
} catch (IOException e) {
throw new RuntimeException("sendError IO fail in DebugServlet", e);
}
}
}
}
@BeforeClass
public static void beforeTest() throws Exception {
JettyConfig jettyConfig = JettyConfig.builder()
.withServlet(new ServletHolder(BasicHttpSolrClientTest.RedirectServlet.class), "/redirect/*")
.withServlet(new ServletHolder(BasicHttpSolrClientTest.SlowServlet.class), "/slow/*")
.withServlet(new ServletHolder(DebugServlet.class), "/debug/*")
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
createAndStartJetty(legacyExampleCollection1SolrHome(), jettyConfig);
}
private Http2SolrClient getHttp2SolrClient(String url, int connectionTimeOut, int socketTimeout) {
return new Http2SolrClient.Builder(url)
.connectionTimeout(connectionTimeOut)
.idleTimeout(socketTimeout)
.build();
}
private Http2SolrClient getHttp2SolrClient(String url) {
return new Http2SolrClient.Builder(url)
.build();
}
@Test
public void testTimeout() throws Exception {
SolrQuery q = new SolrQuery("*:*");
try(Http2SolrClient client = getHttp2SolrClient(jetty.getBaseUrl().toString() + "/slow/foo", DEFAULT_CONNECTION_TIMEOUT, 2000)) {
client.query(q, SolrRequest.METHOD.GET);
fail("No exception thrown.");
} catch (SolrServerException e) {
assertTrue(e.getMessage().contains("timeout") || e.getMessage().contains("Timeout"));
}
}
/**
* test that SolrExceptions thrown by HttpSolrClient can
* correctly encapsulate http status codes even when not on the list of
* ErrorCodes solr may return.
*/
@Test
public void testSolrExceptionCodeNotFromSolr() throws IOException, SolrServerException {
final int status = 527;
assertEquals(status + " didn't generate an UNKNOWN error code, someone modified the list of valid ErrorCode's w/o changing this test to work a different way",
SolrException.ErrorCode.UNKNOWN, SolrException.ErrorCode.getErrorCode(status));
try (Http2SolrClient client = getHttp2SolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) {
DebugServlet.setErrorCode(status);
try {
SolrQuery q = new SolrQuery("foo");
client.query(q, SolrRequest.METHOD.GET);
fail("Didn't get excepted exception from oversided request");
} catch (SolrException e) {
assertEquals("Unexpected exception status code", status, e.code());
}
} finally {
DebugServlet.clear();
}
}
@Test
public void testQuery() throws Exception {
DebugServlet.clear();
try (Http2SolrClient client = getHttp2SolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) {
SolrQuery q = new SolrQuery("foo");
q.setParam("a", "\u1234");
try {
client.query(q, SolrRequest.METHOD.GET);
} catch (ParseException ignored) {}
//default method
assertEquals("get", DebugServlet.lastMethod);
//agent
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
//default wt
assertEquals(1, DebugServlet.parameters.get(CommonParams.WT).length);
assertEquals("javabin", DebugServlet.parameters.get(CommonParams.WT)[0]);
//default version
assertEquals(1, DebugServlet.parameters.get(CommonParams.VERSION).length);
assertEquals(client.getParser().getVersion(), DebugServlet.parameters.get(CommonParams.VERSION)[0]);
//agent
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
//content-type
assertEquals(null, DebugServlet.headers.get("content-type"));
//param encoding
assertEquals(1, DebugServlet.parameters.get("a").length);
assertEquals("\u1234", DebugServlet.parameters.get("a")[0]);
//POST
DebugServlet.clear();
try {
client.query(q, SolrRequest.METHOD.POST);
} catch (ParseException ignored) {}
assertEquals("post", DebugServlet.lastMethod);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
assertEquals(1, DebugServlet.parameters.get(CommonParams.WT).length);
assertEquals("javabin", DebugServlet.parameters.get(CommonParams.WT)[0]);
assertEquals(1, DebugServlet.parameters.get(CommonParams.VERSION).length);
assertEquals(client.getParser().getVersion(), DebugServlet.parameters.get(CommonParams.VERSION)[0]);
assertEquals(1, DebugServlet.parameters.get("a").length);
assertEquals("\u1234", DebugServlet.parameters.get("a")[0]);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
assertEquals("application/x-www-form-urlencoded", DebugServlet.headers.get("content-type"));
//PUT
DebugServlet.clear();
try {
client.query(q, SolrRequest.METHOD.PUT);
} catch (ParseException ignored) {}
assertEquals("put", DebugServlet.lastMethod);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
assertEquals(1, DebugServlet.parameters.get(CommonParams.WT).length);
assertEquals("javabin", DebugServlet.parameters.get(CommonParams.WT)[0]);
assertEquals(1, DebugServlet.parameters.get(CommonParams.VERSION).length);
assertEquals(client.getParser().getVersion(), DebugServlet.parameters.get(CommonParams.VERSION)[0]);
assertEquals(1, DebugServlet.parameters.get("a").length);
assertEquals("\u1234", DebugServlet.parameters.get("a")[0]);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
assertEquals("application/x-www-form-urlencoded", DebugServlet.headers.get("content-type"));
//XML/GET
client.setParser(new XMLResponseParser());
DebugServlet.clear();
try {
client.query(q, SolrRequest.METHOD.GET);
} catch (ParseException ignored) {}
assertEquals("get", DebugServlet.lastMethod);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
assertEquals(1, DebugServlet.parameters.get(CommonParams.WT).length);
assertEquals("xml", DebugServlet.parameters.get(CommonParams.WT)[0]);
assertEquals(1, DebugServlet.parameters.get(CommonParams.VERSION).length);
assertEquals(client.getParser().getVersion(), DebugServlet.parameters.get(CommonParams.VERSION)[0]);
assertEquals(1, DebugServlet.parameters.get("a").length);
assertEquals("\u1234", DebugServlet.parameters.get("a")[0]);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
//XML/POST
client.setParser(new XMLResponseParser());
DebugServlet.clear();
try {
client.query(q, SolrRequest.METHOD.POST);
} catch (ParseException ignored) {}
assertEquals("post", DebugServlet.lastMethod);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
assertEquals(1, DebugServlet.parameters.get(CommonParams.WT).length);
assertEquals("xml", DebugServlet.parameters.get(CommonParams.WT)[0]);
assertEquals(1, DebugServlet.parameters.get(CommonParams.VERSION).length);
assertEquals(client.getParser().getVersion(), DebugServlet.parameters.get(CommonParams.VERSION)[0]);
assertEquals(1, DebugServlet.parameters.get("a").length);
assertEquals("\u1234", DebugServlet.parameters.get("a")[0]);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
assertEquals("application/x-www-form-urlencoded", DebugServlet.headers.get("content-type"));
client.setParser(new XMLResponseParser());
DebugServlet.clear();
try {
client.query(q, SolrRequest.METHOD.PUT);
} catch (ParseException ignored) {}
assertEquals("put", DebugServlet.lastMethod);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
assertEquals(1, DebugServlet.parameters.get(CommonParams.WT).length);
assertEquals("xml", DebugServlet.parameters.get(CommonParams.WT)[0]);
assertEquals(1, DebugServlet.parameters.get(CommonParams.VERSION).length);
assertEquals(client.getParser().getVersion(), DebugServlet.parameters.get(CommonParams.VERSION)[0]);
assertEquals(1, DebugServlet.parameters.get("a").length);
assertEquals("\u1234", DebugServlet.parameters.get("a")[0]);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
assertEquals("application/x-www-form-urlencoded", DebugServlet.headers.get("content-type"));
}
}
@Test
public void testDelete() throws Exception {
DebugServlet.clear();
try (Http2SolrClient client = getHttp2SolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) {
try {
client.deleteById("id");
} catch (ParseException ignored) {}
//default method
assertEquals("post", DebugServlet.lastMethod);
//agent
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
//default wt
assertEquals(1, DebugServlet.parameters.get(CommonParams.WT).length);
assertEquals("javabin", DebugServlet.parameters.get(CommonParams.WT)[0]);
//default version
assertEquals(1, DebugServlet.parameters.get(CommonParams.VERSION).length);
assertEquals(client.getParser().getVersion(), DebugServlet.parameters.get(CommonParams.VERSION)[0]);
//agent
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
//XML
client.setParser(new XMLResponseParser());
try {
client.deleteByQuery("*:*");
} catch (ParseException ignored) {}
assertEquals("post", DebugServlet.lastMethod);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
assertEquals(1, DebugServlet.parameters.get(CommonParams.WT).length);
assertEquals("xml", DebugServlet.parameters.get(CommonParams.WT)[0]);
assertEquals(1, DebugServlet.parameters.get(CommonParams.VERSION).length);
assertEquals(client.getParser().getVersion(), DebugServlet.parameters.get(CommonParams.VERSION)[0]);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
}
}
@Test
public void testGetById() throws Exception {
DebugServlet.clear();
try (Http2SolrClient client = getHttp2SolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) {
Collection<String> ids = Collections.singletonList("a");
try {
client.getById("a");
} catch (ParseException ignored) {}
try {
client.getById(ids, null);
} catch (ParseException ignored) {}
try {
client.getById("foo", "a");
} catch (ParseException ignored) {}
try {
client.getById("foo", ids, null);
} catch (ParseException ignored) {}
}
}
@Test
public void testUpdate() throws Exception {
DebugServlet.clear();
try (Http2SolrClient client = getHttp2SolrClient(jetty.getBaseUrl().toString() + "/debug/foo")) {
UpdateRequest req = new UpdateRequest();
req.add(new SolrInputDocument());
req.setParam("a", "\u1234");
try {
client.request(req);
} catch (ParseException ignored) {}
//default method
assertEquals("post", DebugServlet.lastMethod);
//agent
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
//default wt
assertEquals(1, DebugServlet.parameters.get(CommonParams.WT).length);
assertEquals("javabin", DebugServlet.parameters.get(CommonParams.WT)[0]);
//default version
assertEquals(1, DebugServlet.parameters.get(CommonParams.VERSION).length);
assertEquals(client.getParser().getVersion(), DebugServlet.parameters.get(CommonParams.VERSION)[0]);
//content type
assertEquals("application/javabin", DebugServlet.headers.get("content-type"));
//parameter encoding
assertEquals(1, DebugServlet.parameters.get("a").length);
assertEquals("\u1234", DebugServlet.parameters.get("a")[0]);
//XML response and writer
client.setParser(new XMLResponseParser());
client.setRequestWriter(new RequestWriter());
try {
client.request(req);
} catch (ParseException ignored) {}
assertEquals("post", DebugServlet.lastMethod);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
assertEquals(1, DebugServlet.parameters.get(CommonParams.WT).length);
assertEquals("xml", DebugServlet.parameters.get(CommonParams.WT)[0]);
assertEquals(1, DebugServlet.parameters.get(CommonParams.VERSION).length);
assertEquals(client.getParser().getVersion(), DebugServlet.parameters.get(CommonParams.VERSION)[0]);
assertEquals("application/xml; charset=UTF-8", DebugServlet.headers.get("content-type"));
assertEquals(1, DebugServlet.parameters.get("a").length);
assertEquals("\u1234", DebugServlet.parameters.get("a")[0]);
//javabin request
client.setParser(new BinaryResponseParser());
client.setRequestWriter(new BinaryRequestWriter());
DebugServlet.clear();
try {
client.request(req);
} catch (ParseException ignored) {}
assertEquals("post", DebugServlet.lastMethod);
assertEquals(EXPECTED_USER_AGENT, DebugServlet.headers.get("user-agent"));
assertEquals(1, DebugServlet.parameters.get(CommonParams.WT).length);
assertEquals("javabin", DebugServlet.parameters.get(CommonParams.WT)[0]);
assertEquals(1, DebugServlet.parameters.get(CommonParams.VERSION).length);
assertEquals(client.getParser().getVersion(), DebugServlet.parameters.get(CommonParams.VERSION)[0]);
assertEquals("application/javabin", DebugServlet.headers.get("content-type"));
assertEquals(1, DebugServlet.parameters.get("a").length);
assertEquals("\u1234", DebugServlet.parameters.get("a")[0]);
}
}
@Test
public void testRedirect() throws Exception {
final String clientUrl = jetty.getBaseUrl().toString() + "/redirect/foo";
try (Http2SolrClient client = getHttp2SolrClient(clientUrl)) {
SolrQuery q = new SolrQuery("*:*");
// default = false
try {
client.query(q);
fail("Should have thrown an exception.");
} catch (SolrServerException e) {
assertTrue(e.getMessage().contains("redirect"));
}
client.setFollowRedirects(true);
client.query(q);
//And back again:
client.setFollowRedirects(false);
try {
client.query(q);
fail("Should have thrown an exception.");
} catch (SolrServerException e) {
assertTrue(e.getMessage().contains("redirect"));
}
}
}
@Test
public void testCollectionParameters() throws IOException, SolrServerException {
try (Http2SolrClient client = getHttp2SolrClient(jetty.getBaseUrl().toString())) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", "collection");
client.add("collection1", doc);
client.commit("collection1");
assertEquals(1, client.query("collection1", new SolrQuery("id:collection")).getResults().getNumFound());
}
final String collection1Url = jetty.getBaseUrl().toString() + "/collection1";
try (Http2SolrClient client = getHttp2SolrClient(collection1Url)) {
assertEquals(1, client.query(new SolrQuery("id:collection")).getResults().getNumFound());
}
}
private Set<String> setOf(String... keys) {
Set<String> set = new TreeSet<>();
if (keys != null) {
Collections.addAll(set, keys);
}
return set;
}
private void setReqParamsOf(UpdateRequest req, String... keys) {
if (keys != null) {
for (String k : keys) {
req.setParam(k, k+"Value");
}
}
}
private void verifyServletState(Http2SolrClient client, SolrRequest request) {
// check query String
Iterator<String> paramNames = request.getParams().getParameterNamesIterator();
while (paramNames.hasNext()) {
String name = paramNames.next();
String [] values = request.getParams().getParams(name);
if (values != null) {
for (String value : values) {
boolean shouldBeInQueryString = client.getQueryParams().contains(name)
|| (request.getQueryParams() != null && request.getQueryParams().contains(name));
assertEquals(shouldBeInQueryString, DebugServlet.queryString.contains(name + "=" + value));
// in either case, it should be in the parameters
assertNotNull(DebugServlet.parameters.get(name));
assertEquals(1, DebugServlet.parameters.get(name).length);
assertEquals(value, DebugServlet.parameters.get(name)[0]);
}
}
}
}
@Test
public void testQueryString() throws Exception {
final String clientUrl = jetty.getBaseUrl().toString() + "/debug/foo";
try(Http2SolrClient client = getHttp2SolrClient(clientUrl)) {
// test without request query params
DebugServlet.clear();
client.setQueryParams(setOf("serverOnly"));
UpdateRequest req = new UpdateRequest();
setReqParamsOf(req, "serverOnly", "notServer");
try {
client.request(req);
} catch (ParseException ignored) {}
verifyServletState(client, req);
// test without server query params
DebugServlet.clear();
client.setQueryParams(setOf());
req = new UpdateRequest();
req.setQueryParams(setOf("requestOnly"));
setReqParamsOf(req, "requestOnly", "notRequest");
try {
client.request(req);
} catch (ParseException ignored) {}
verifyServletState(client, req);
// test with both request and server query params
DebugServlet.clear();
req = new UpdateRequest();
client.setQueryParams(setOf("serverOnly", "both"));
req.setQueryParams(setOf("requestOnly", "both"));
setReqParamsOf(req, "serverOnly", "requestOnly", "both", "neither");
try {
client.request(req);
} catch (ParseException ignored) {}
verifyServletState(client, req);
// test with both request and server query params with single stream
DebugServlet.clear();
req = new UpdateRequest();
req.add(new SolrInputDocument());
client.setQueryParams(setOf("serverOnly", "both"));
req.setQueryParams(setOf("requestOnly", "both"));
setReqParamsOf(req, "serverOnly", "requestOnly", "both", "neither");
try {
client.request(req);
} catch (ParseException ignored) {}
// NOTE: single stream requests send all the params
// as part of the query string. So add "neither" to the request
// so it passes the verification step.
req.setQueryParams(setOf("requestOnly", "both", "neither"));
verifyServletState(client, req);
}
}
/**
* Missed tests :
* - set cookies via interceptor
* - invariant params
* - compression
* - get raw stream
*/
}

2
solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBadInputTest.java
View File

@ -40,7 +40,7 @@ public class HttpSolrClientBadInputTest extends SolrJettyTestBase {
@BeforeClass
public static void beforeTest() throws Exception {
JettyConfig jettyConfig = JettyConfig.builder()
.withSSLConfig(sslConfig)
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
createAndStartJetty(legacyExampleCollection1SolrHome(), jettyConfig);
}

2
solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBadInputTest.java
View File

@ -39,7 +39,7 @@ public class LBHttpSolrClientBadInputTest extends SolrJettyTestBase {
@BeforeClass
public static void beforeTest() throws Exception {
JettyConfig jettyConfig = JettyConfig.builder()
.withSSLConfig(sslConfig)
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
createAndStartJetty(legacyExampleCollection1SolrHome(), jettyConfig);
}

17
solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
View File

@ -53,7 +53,6 @@ import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
@ -361,7 +360,11 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {
j.start();
jettys.add(j);
clients.add(createNewSolrClient(j.getLocalPort()));
String shardStr = buildUrl(j.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME;
String shardStr = buildUrl(j.getLocalPort());
if (shardStr.endsWith("/")) shardStr += DEFAULT_TEST_CORENAME;
else shardStr += "/" + DEFAULT_TEST_CORENAME;
shardsArr[i] = shardStr;
sb.append(shardStr);
}
@ -468,7 +471,7 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {
.setContext(context)
.withFilters(getExtraRequestFilters())
.withServlets(getExtraServlets())
.withSSLConfig(sslConfig)
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build());
return jetty;
@ -487,8 +490,12 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {
protected SolrClient createNewSolrClient(int port) {
try {
// setup the client...
HttpSolrClient client = getHttpSolrClient(buildUrl(port) + "/" + DEFAULT_TEST_CORENAME);
return client;
String baseUrl = buildUrl(port);
if (baseUrl.endsWith("/")) {
return getHttpSolrClient(baseUrl + DEFAULT_TEST_CORENAME);
} else {
return getHttpSolrClient(baseUrl + "/" + DEFAULT_TEST_CORENAME);
}
}
catch (Exception ex) {
throw new RuntimeException(ex);

16
solr/test-framework/src/java/org/apache/solr/SolrJettyTestBase.java
View File

@ -65,7 +65,7 @@ abstract public class SolrJettyTestBase extends SolrTestCaseJ4
.setContext(context)
.stopAtShutdown(stopAtShutdown)
.withServlets(extraServlets)
.withSSLConfig(sslConfig)
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
Properties nodeProps = new Properties();
@ -89,7 +89,7 @@ abstract public class SolrJettyTestBase extends SolrTestCaseJ4
}
public static JettySolrRunner createAndStartJetty(String solrHome) throws Exception {
return createAndStartJetty(solrHome, new Properties(), JettyConfig.builder().withSSLConfig(sslConfig).build());
return createAndStartJetty(solrHome, new Properties(), JettyConfig.builder().withSSLConfig(sslConfig.buildServerSSLConfig()).build());
}
public static JettySolrRunner createAndStartJetty(String solrHome, Properties nodeProperties, JettyConfig jettyConfig) throws Exception {
@ -118,7 +118,7 @@ abstract public class SolrJettyTestBase extends SolrTestCaseJ4
}
@After
public void afterClass() throws Exception {
public synchronized void afterClass() throws Exception {
if (client != null) client.close();
client = null;
}
@ -132,13 +132,11 @@ abstract public class SolrJettyTestBase extends SolrTestCaseJ4
}
public SolrClient getSolrClient() {
{
if (client == null) {
client = createNewSolrClient();
}
return client;
public synchronized SolrClient getSolrClient() {
if (client == null) {
client = createNewSolrClient();
}
return client;
}
/**

2
solr/test-framework/src/java/org/apache/solr/SolrTestCaseHS.java
View File

@ -458,7 +458,7 @@ public class SolrTestCaseHS extends SolrTestCaseJ4 {
.stopAtShutdown(true)
.setContext("/solr")
.setPort(port)
.withSSLConfig(sslConfig)
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
Properties nodeProperties = new Properties();
nodeProperties.setProperty("solrconfig", solrconfigFile);

9
solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
View File

@ -89,6 +89,7 @@ import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
@ -171,7 +172,7 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactor
@SuppressSysoutChecks(bugUrl = "Solr dumps tons of logs to console.")
@SuppressFileSystems("ExtrasFS") // might be ok, the failures with e.g. nightly runs might be "normal"
@RandomizeSSL()
@ThreadLeakLingering(linger = 3000)
@ThreadLeakLingering(linger = 10000)
public abstract class SolrTestCaseJ4 extends LuceneTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ -287,10 +288,11 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
startTrackingSearchers();
ignoreException("ignore_exception");
newRandomConfig();
sslConfig = buildSSLConfig();
// based on randomized SSL config, set SchemaRegistryProvider appropriately
HttpClientUtil.setSchemaRegistryProvider(sslConfig.buildClientSchemaRegistryProvider());
Http2SolrClient.setDefaultSSLConfig(sslConfig.buildClientSSLConfig());
if(isSSLMode()) {
// SolrCloud tests should usually clear this
System.setProperty("urlScheme", "https");
@ -335,6 +337,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
System.clearProperty("solr.cloud.wait-for-updates-with-stale-state-pause");
System.clearProperty("solr.zkclienttmeout");
HttpClientUtil.resetHttpClientBuilder();
Http2SolrClient.resetSslContextFactory();
clearNumericTypesProperties();
@ -487,7 +490,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
}
protected static JettyConfig buildJettyConfig(String context) {
return JettyConfig.builder().setContext(context).withSSLConfig(sslConfig).build();
return JettyConfig.builder().setContext(context).withSSLConfig(sslConfig.buildServerSSLConfig()).build();
}
protected static String buildUrl(final int port, final String context) {

28
solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
View File

@ -620,6 +620,30 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
return cnt;
}
public JettySolrRunner createJetty(String dataDir, String ulogDir, String shardList,
String solrConfigOverride) throws Exception {
JettyConfig jettyconfig = JettyConfig.builder()
.setContext(context)
.stopAtShutdown(false)
.withServlets(getExtraServlets())
.withFilters(getExtraRequestFilters())
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
Properties props = new Properties();
props.setProperty("solr.data.dir", getDataDir(dataDir));
props.setProperty("shards", shardList);
props.setProperty("solr.ulog.dir", ulogDir);
props.setProperty("solrconfig", solrConfigOverride);
JettySolrRunner jetty = new JettySolrRunner(getSolrHome(), props, jettyconfig);
jetty.start();
return jetty;
}
public final JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, String solrConfigOverride, String schemaOverride) throws Exception {
return createJetty(solrHome, dataDir, shardList, solrConfigOverride, schemaOverride, null);
}
@ -635,7 +659,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
.stopAtShutdown(false)
.withServlets(getExtraServlets())
.withFilters(getExtraRequestFilters())
.withSSLConfig(sslConfig)
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
Properties props = new Properties();
@ -673,7 +697,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
.stopAtShutdown(false)
.withServlets(getExtraServlets())
.withFilters(getExtraRequestFilters())
.withSSLConfig(sslConfig)
.withSSLConfig(sslConfig.buildServerSSLConfig())
.build();
Properties props = new Properties();

Some files were not shown because too many files have changed in this diff Show More