From 8278886966c6da7379cf9c9505f7859b832c4ab3 Mon Sep 17 00:00:00 2001 From: erick Date: Fri, 13 Dec 2019 10:01:37 -0500 Subject: [PATCH 01/15] SOLR-14026: Upgrade Jetty to 9.4.24.v20191120 and dropwizard to 4.1.2 --- lucene/ivy-versions.properties | 4 ++-- .../jetty-continuation-9.4.19.v20190610.jar.sha1 | 1 - .../jetty-continuation-9.4.24.v20191120.jar.sha1 | 1 + lucene/licenses/jetty-http-9.4.19.v20190610.jar.sha1 | 1 - lucene/licenses/jetty-http-9.4.24.v20191120.jar.sha1 | 1 + lucene/licenses/jetty-io-9.4.19.v20190610.jar.sha1 | 1 - lucene/licenses/jetty-io-9.4.24.v20191120.jar.sha1 | 1 + .../licenses/jetty-server-9.4.19.v20190610.jar.sha1 | 1 - .../licenses/jetty-server-9.4.24.v20191120.jar.sha1 | 1 + .../licenses/jetty-servlet-9.4.19.v20190610.jar.sha1 | 1 - .../licenses/jetty-servlet-9.4.24.v20191120.jar.sha1 | 1 + lucene/licenses/jetty-util-9.4.19.v20190610.jar.sha1 | 1 - lucene/licenses/jetty-util-9.4.24.v20191120.jar.sha1 | 1 + solr/CHANGES.txt | 6 ++++-- .../solr/metrics/reporters/SolrSlf4jReporter.java | 2 +- .../solr/metrics/reporters/solr/SolrReporter.java | 8 +++++++- .../solr/handler/admin/CoreAdminHandlerTest.java | 11 ++++++----- .../solr/security/JWTAuthPluginIntegrationTest.java | 3 +-- solr/licenses/http2-client-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/http2-client-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/http2-common-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/http2-common-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/http2-hpack-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/http2-hpack-9.4.24.v20191120.jar.sha1 | 1 + ...p2-http-client-transport-9.4.19.v20190610.jar.sha1 | 1 - ...p2-http-client-transport-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/http2-server-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/http2-server-9.4.24.v20191120.jar.sha1 | 1 + .../jetty-alpn-client-9.4.19.v20190610.jar.sha1 | 1 - .../jetty-alpn-client-9.4.24.v20191120.jar.sha1 | 1 + .../jetty-alpn-java-client-9.4.19.v20190610.jar.sha1 | 1 - .../jetty-alpn-java-client-9.4.24.v20191120.jar.sha1 | 1 + .../jetty-alpn-java-server-9.4.19.v20190610.jar.sha1 | 1 - .../jetty-alpn-java-server-9.4.24.v20191120.jar.sha1 | 1 + .../jetty-alpn-server-9.4.19.v20190610.jar.sha1 | 1 - .../jetty-alpn-server-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/jetty-client-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/jetty-client-9.4.24.v20191120.jar.sha1 | 1 + .../jetty-continuation-9.4.19.v20190610.jar.sha1 | 1 - .../jetty-continuation-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/jetty-deploy-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/jetty-deploy-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/jetty-http-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/jetty-http-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/jetty-io-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/jetty-io-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/jetty-jmx-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/jetty-jmx-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/jetty-rewrite-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/jetty-rewrite-9.4.24.v20191120.jar.sha1 | 1 + .../licenses/jetty-security-9.4.19.v20190610.jar.sha1 | 1 - .../licenses/jetty-security-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/jetty-server-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/jetty-server-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/jetty-servlet-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/jetty-servlet-9.4.24.v20191120.jar.sha1 | 1 + .../licenses/jetty-servlets-9.4.19.v20190610.jar.sha1 | 1 - .../licenses/jetty-servlets-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/jetty-util-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/jetty-util-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/jetty-webapp-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/jetty-webapp-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/jetty-xml-9.4.19.v20190610.jar.sha1 | 1 - solr/licenses/jetty-xml-9.4.24.v20191120.jar.sha1 | 1 + solr/licenses/metrics-core-4.0.5.jar.sha1 | 1 - solr/licenses/metrics-core-4.1.2.jar.sha1 | 1 + solr/licenses/metrics-graphite-4.0.5.jar.sha1 | 1 - solr/licenses/metrics-graphite-4.1.2.jar.sha1 | 1 + solr/licenses/metrics-jetty9-4.0.5.jar.sha1 | 1 - solr/licenses/metrics-jetty9-4.1.2.jar.sha1 | 1 + solr/licenses/metrics-jmx-4.0.5.jar.sha1 | 1 - solr/licenses/metrics-jmx-4.1.2.jar.sha1 | 1 + solr/licenses/metrics-jvm-4.0.5.jar.sha1 | 1 - solr/licenses/metrics-jvm-4.1.2.jar.sha1 | 1 + solr/licenses/start.jar.sha1 | 2 +- 75 files changed, 56 insertions(+), 48 deletions(-) delete mode 100644 lucene/licenses/jetty-continuation-9.4.19.v20190610.jar.sha1 create mode 100644 lucene/licenses/jetty-continuation-9.4.24.v20191120.jar.sha1 delete mode 100644 lucene/licenses/jetty-http-9.4.19.v20190610.jar.sha1 create mode 100644 lucene/licenses/jetty-http-9.4.24.v20191120.jar.sha1 delete mode 100644 lucene/licenses/jetty-io-9.4.19.v20190610.jar.sha1 create mode 100644 lucene/licenses/jetty-io-9.4.24.v20191120.jar.sha1 delete mode 100644 lucene/licenses/jetty-server-9.4.19.v20190610.jar.sha1 create mode 100644 lucene/licenses/jetty-server-9.4.24.v20191120.jar.sha1 delete mode 100644 lucene/licenses/jetty-servlet-9.4.19.v20190610.jar.sha1 create mode 100644 lucene/licenses/jetty-servlet-9.4.24.v20191120.jar.sha1 delete mode 100644 lucene/licenses/jetty-util-9.4.19.v20190610.jar.sha1 create mode 100644 lucene/licenses/jetty-util-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/http2-client-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/http2-client-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/http2-common-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/http2-common-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/http2-hpack-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/http2-hpack-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/http2-http-client-transport-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/http2-http-client-transport-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/http2-server-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/http2-server-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-alpn-client-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-alpn-client-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-alpn-java-client-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-alpn-java-client-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-alpn-java-server-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-alpn-java-server-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-alpn-server-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-alpn-server-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-client-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-client-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-continuation-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-continuation-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-deploy-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-deploy-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-http-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-http-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-io-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-io-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-jmx-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-jmx-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-rewrite-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-rewrite-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-security-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-security-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-server-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-server-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-servlet-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-servlet-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-servlets-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-servlets-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-util-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-util-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-webapp-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-webapp-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/jetty-xml-9.4.19.v20190610.jar.sha1 create mode 100644 solr/licenses/jetty-xml-9.4.24.v20191120.jar.sha1 delete mode 100644 solr/licenses/metrics-core-4.0.5.jar.sha1 create mode 100644 solr/licenses/metrics-core-4.1.2.jar.sha1 delete mode 100644 solr/licenses/metrics-graphite-4.0.5.jar.sha1 create mode 100644 solr/licenses/metrics-graphite-4.1.2.jar.sha1 delete mode 100644 solr/licenses/metrics-jetty9-4.0.5.jar.sha1 create mode 100644 solr/licenses/metrics-jetty9-4.1.2.jar.sha1 delete mode 100644 solr/licenses/metrics-jmx-4.0.5.jar.sha1 create mode 100644 solr/licenses/metrics-jmx-4.1.2.jar.sha1 delete mode 100644 solr/licenses/metrics-jvm-4.0.5.jar.sha1 create mode 100644 solr/licenses/metrics-jvm-4.1.2.jar.sha1 diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index d0961539153..ed2593accf4 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -62,7 +62,7 @@ com.sun.jersey.version = 1.19 /commons-logging/commons-logging = 1.1.3 /de.l3s.boilerpipe/boilerpipe = 1.1.0 -io.dropwizard.metrics.version = 4.0.5 +io.dropwizard.metrics.version = 4.1.2 /io.dropwizard.metrics/metrics-core = ${io.dropwizard.metrics.version} /io.dropwizard.metrics/metrics-graphite = ${io.dropwizard.metrics.version} /io.dropwizard.metrics/metrics-jetty9 = ${io.dropwizard.metrics.version} @@ -260,7 +260,7 @@ org.codehaus.janino.version = 3.0.9 /org.codehaus.woodstox/stax2-api = 3.1.4 /org.codehaus.woodstox/woodstox-core-asl = 4.4.1 -org.eclipse.jetty.version = 9.4.19.v20190610 +org.eclipse.jetty.version = 9.4.24.v20191120 /org.eclipse.jetty.http2/http2-client = ${org.eclipse.jetty.version} /org.eclipse.jetty.http2/http2-common = ${org.eclipse.jetty.version} /org.eclipse.jetty.http2/http2-hpack = ${org.eclipse.jetty.version} diff --git a/lucene/licenses/jetty-continuation-9.4.19.v20190610.jar.sha1 b/lucene/licenses/jetty-continuation-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index d32802987d7..00000000000 --- a/lucene/licenses/jetty-continuation-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4acddfa41f45790e43fe4be257c3c4bcf6b846ff diff --git a/lucene/licenses/jetty-continuation-9.4.24.v20191120.jar.sha1 b/lucene/licenses/jetty-continuation-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..54d85735811 --- /dev/null +++ b/lucene/licenses/jetty-continuation-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +69125cf74b07f1b9d60b5c94da47cb04c098f654 diff --git a/lucene/licenses/jetty-http-9.4.19.v20190610.jar.sha1 b/lucene/licenses/jetty-http-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 76f1249a504..00000000000 --- a/lucene/licenses/jetty-http-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b59ff8ecb0cf5d6234958f2404eabf0b72464e14 diff --git a/lucene/licenses/jetty-http-9.4.24.v20191120.jar.sha1 b/lucene/licenses/jetty-http-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..93c2d9afc19 --- /dev/null +++ b/lucene/licenses/jetty-http-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +d3f0b0fb016ef8d35ffb199d928ffbcbfa121c86 diff --git a/lucene/licenses/jetty-io-9.4.19.v20190610.jar.sha1 b/lucene/licenses/jetty-io-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 5af4939f6a3..00000000000 --- a/lucene/licenses/jetty-io-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7eb9a6be62d84e1691e5fdc99223e632485619a8 diff --git a/lucene/licenses/jetty-io-9.4.24.v20191120.jar.sha1 b/lucene/licenses/jetty-io-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..1579180a83c --- /dev/null +++ b/lucene/licenses/jetty-io-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +dcb6d4d505ef74898e3a64a38c40195c01e97119 diff --git a/lucene/licenses/jetty-server-9.4.19.v20190610.jar.sha1 b/lucene/licenses/jetty-server-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index cbfdf11ae38..00000000000 --- a/lucene/licenses/jetty-server-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -55786f6e6649bd49425a7da1ac72cd85b8dd4bef diff --git a/lucene/licenses/jetty-server-9.4.24.v20191120.jar.sha1 b/lucene/licenses/jetty-server-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..ffd2c58fd5c --- /dev/null +++ b/lucene/licenses/jetty-server-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +7885cc3d5d7701a444acada7ab97f89846514875 diff --git a/lucene/licenses/jetty-servlet-9.4.19.v20190610.jar.sha1 b/lucene/licenses/jetty-servlet-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index c36dab5c770..00000000000 --- a/lucene/licenses/jetty-servlet-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -89e25610b3199fdf34a831c1b306f7e765928959 diff --git a/lucene/licenses/jetty-servlet-9.4.24.v20191120.jar.sha1 b/lucene/licenses/jetty-servlet-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..3ad349e3c0a --- /dev/null +++ b/lucene/licenses/jetty-servlet-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +ca1803fde51b795c0a8346ca8bc6277d9d04d01d diff --git a/lucene/licenses/jetty-util-9.4.19.v20190610.jar.sha1 b/lucene/licenses/jetty-util-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 0a2b5ade4f3..00000000000 --- a/lucene/licenses/jetty-util-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2fd3cd40279280e8c56241f753d2c52d8d446d19 diff --git a/lucene/licenses/jetty-util-9.4.24.v20191120.jar.sha1 b/lucene/licenses/jetty-util-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..a02b48126d5 --- /dev/null +++ b/lucene/licenses/jetty-util-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +3095acb088f4ff9e3fd9aedf98db73e3c18ea849 diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 944ba095358..5811218f5c4 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -101,7 +101,7 @@ Apache Tika 1.19.1 Carrot2 3.16.0 Velocity 2.0 and Velocity Tools 3.0 Apache ZooKeeper 3.5.5 -Jetty 9.4.19.v20190610 +Jetty 9.4.24.v20191120 Upgrade Notes --------------------- @@ -179,6 +179,8 @@ Upgrade Notes * SOLR-14065: VelocityResponseWriter has been deprecated and may be removed in a future version. +* SOLR-14026: Upgrade Jetty to 9.4.24.v20191120 and dropwizard to 4.1.2 (Erick Erickson) + New Features --------------------- * SOLR-13821: A Package store to store and load package artifacts (noble, Ishan Chattopadhyaya) @@ -351,7 +353,7 @@ Apache Tika 1.19.1 Carrot2 3.16.0 Velocity 2.0 and Velocity Tools 3.0 Apache ZooKeeper 3.5.5 -Jetty 9.4.19.v20190610 +Jetty 9.4.24.v20191120 Upgrade Notes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java index e65d1193c2b..aaf37c58c84 100644 --- a/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java +++ b/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java @@ -67,7 +67,7 @@ public class SolrSlf4jReporter extends FilteringSolrMetricReporter { final Map mdcContext; Slf4jReporterWrapper(String logger, Map mdcContext, Slf4jReporter delegate, TimeUnit rateUnit, TimeUnit durationUnit) { - super(null, logger, null, rateUnit, durationUnit); + super(metricManager.registry(registryName), logger, null, rateUnit, durationUnit); this.delegate = delegate; this.mdcContext = mdcContext; } diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java index 5a83cd9fa2d..59e25d402ff 100644 --- a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java +++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java @@ -37,6 +37,7 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricFilter; +import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.ScheduledReporter; import com.codahale.metrics.Timer; import org.apache.http.client.HttpClient; @@ -301,12 +302,17 @@ public class SolrReporter extends ScheduledReporter { } } + // Recent dropwizard (found with version 4.1.2) requires that you _must_ call the superclass with a non-null registry. + // We delegate to registries anyway, so having a dummy registry is harmless. + private static final MetricRegistry dummyRegistry = new MetricRegistry(); + public SolrReporter(HttpClient httpClient, Supplier urlProvider, SolrMetricManager metricManager, List metrics, String handler, String reporterId, TimeUnit rateUnit, TimeUnit durationUnit, SolrParams params, boolean skipHistograms, boolean skipAggregateValues, boolean cloudClient, boolean compact) { - super(null, "solr-reporter", MetricFilter.ALL, rateUnit, durationUnit, null, true); + super(dummyRegistry, "solr-reporter", MetricFilter.ALL, rateUnit, durationUnit, null, true); + this.metricManager = metricManager; this.urlProvider = urlProvider; this.reporterId = reporterId; diff --git a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java index e9c09da2bcb..eb3d1ebf86c 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java @@ -32,6 +32,7 @@ import org.apache.lucene.util.Constants; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.request.CoreStatus; @@ -190,12 +191,12 @@ public class CoreAdminHandlerTest extends SolrTestCaseJ4 { CoreAdminParams.CoreAdminAction.STATUS.toString(), CoreAdminParams.CORE, "bogus_dir_core"), resp); - Map failures = + @SuppressWarnings("unchecked") + Map failures = (Map) resp.getValues().get("initFailures"); assertNotNull("core failures is null", failures); - NamedList status = - (NamedList)resp.getValues().get("status"); + NamedList status = (NamedList)resp.getValues().get("status"); assertNotNull("core status is null", status); assertEquals("wrong number of core failures", 1, failures.size()); @@ -338,7 +339,7 @@ public class CoreAdminHandlerTest extends SolrTestCaseJ4 { req.process(client); } - HttpSolrClient.RemoteSolrException rse = expectThrows(HttpSolrClient.RemoteSolrException.class, () -> { + BaseHttpSolrClient.RemoteSolrException rse = expectThrows(BaseHttpSolrClient.RemoteSolrException.class, () -> { try (HttpSolrClient client = getHttpSolrClient(runner.getBaseUrl() + "/corex", DEFAULT_CONNECTION_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT * 1000)) { client.query(new SolrQuery("id:*")); @@ -346,7 +347,7 @@ public class CoreAdminHandlerTest extends SolrTestCaseJ4 { runner.stop(); } }); - assertTrue(rse.getMessage(), rse.getMessage().contains("Problem accessing /solr/corex/select")); + assertEquals("Should have received a 404 error", 404, rse.code()); } @Test diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index 20dc667c6ac..76d3de458a2 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -135,8 +135,7 @@ public class JWTAuthPluginIntegrationTest extends SolrCloudAuthTestCase { @Test public void infoRequestValidateXSolrAuthHeaders() throws IOException { Map headers = getHeaders(baseUrl + "/admin/info/system", null); - assertEquals("401", headers.get("code")); - assertEquals("HTTP/1.1 401 Require authentication", headers.get(null)); + assertEquals("Should have received 401 code", "401", headers.get("code")); assertEquals("Bearer realm=\"my-solr-jwt\"", headers.get("WWW-Authenticate")); String authData = new String(Base64.base64ToByteArray(headers.get("X-Solr-AuthData")), UTF_8); assertEquals("{\n" + diff --git a/solr/licenses/http2-client-9.4.19.v20190610.jar.sha1 b/solr/licenses/http2-client-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index c17735d7829..00000000000 --- a/solr/licenses/http2-client-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -da335ee2e7d1439dcc7e11e89941edfad91e9e10 diff --git a/solr/licenses/http2-client-9.4.24.v20191120.jar.sha1 b/solr/licenses/http2-client-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..4118e2f3998 --- /dev/null +++ b/solr/licenses/http2-client-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +2f043b2b3b9d27c17f2a067521dfb69b41fea1b8 diff --git a/solr/licenses/http2-common-9.4.19.v20190610.jar.sha1 b/solr/licenses/http2-common-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index dcfb3b15248..00000000000 --- a/solr/licenses/http2-common-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3b8c59c68d52a3d0de0d53f5b3588be3a5c05fb8 diff --git a/solr/licenses/http2-common-9.4.24.v20191120.jar.sha1 b/solr/licenses/http2-common-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..dd72ecf74f1 --- /dev/null +++ b/solr/licenses/http2-common-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +81ac98f3be6a902e39e3f48496c9790dd02d4950 diff --git a/solr/licenses/http2-hpack-9.4.19.v20190610.jar.sha1 b/solr/licenses/http2-hpack-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index b2092f1894b..00000000000 --- a/solr/licenses/http2-hpack-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3aaf2c8c9c781f10d4d9da6120c5195b2fcb2ad9 diff --git a/solr/licenses/http2-hpack-9.4.24.v20191120.jar.sha1 b/solr/licenses/http2-hpack-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..984e74624a2 --- /dev/null +++ b/solr/licenses/http2-hpack-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +916f481032995159d062ffc44f566891872d8a07 diff --git a/solr/licenses/http2-http-client-transport-9.4.19.v20190610.jar.sha1 b/solr/licenses/http2-http-client-transport-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 366440fabe0..00000000000 --- a/solr/licenses/http2-http-client-transport-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -95f58cd0cfa0c4553fc3901138cc6a03ece23b94 diff --git a/solr/licenses/http2-http-client-transport-9.4.24.v20191120.jar.sha1 b/solr/licenses/http2-http-client-transport-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..ef199c8bc18 --- /dev/null +++ b/solr/licenses/http2-http-client-transport-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +cde2b06c3134600309061a84759d1ef9087a7348 diff --git a/solr/licenses/http2-server-9.4.19.v20190610.jar.sha1 b/solr/licenses/http2-server-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 14875851dc8..00000000000 --- a/solr/licenses/http2-server-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -672891a1abbeef85192d137192e347872a6fc9c3 diff --git a/solr/licenses/http2-server-9.4.24.v20191120.jar.sha1 b/solr/licenses/http2-server-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..a8ed696dca2 --- /dev/null +++ b/solr/licenses/http2-server-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +41365c22bc6046af6e1e10f1be0b4dbfe49902be diff --git a/solr/licenses/jetty-alpn-client-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-alpn-client-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index b2fd01b2a39..00000000000 --- a/solr/licenses/jetty-alpn-client-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8c9283b8a04056a0fced23fc474e62aa39764c6b diff --git a/solr/licenses/jetty-alpn-client-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-alpn-client-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..d077aba9c11 --- /dev/null +++ b/solr/licenses/jetty-alpn-client-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +4aa2da175202a62d62850ade7fb26a64fd451bc2 diff --git a/solr/licenses/jetty-alpn-java-client-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-alpn-java-client-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index f2e0fc27bac..00000000000 --- a/solr/licenses/jetty-alpn-java-client-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -37eff0bd068adca090e14a0fbd9de258a871f9d9 diff --git a/solr/licenses/jetty-alpn-java-client-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-alpn-java-client-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..2272f6ba047 --- /dev/null +++ b/solr/licenses/jetty-alpn-java-client-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +d3e23487151f5393bdcef5449407c5ce29718cdc diff --git a/solr/licenses/jetty-alpn-java-server-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-alpn-java-server-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 860c2190627..00000000000 --- a/solr/licenses/jetty-alpn-java-server-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7e7f62c2c03b74e59211eeeba0ddc067ad422ff7 diff --git a/solr/licenses/jetty-alpn-java-server-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-alpn-java-server-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..55d3dace36e --- /dev/null +++ b/solr/licenses/jetty-alpn-java-server-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +7a999fbfb9905465c1494052b612b4a4bbb349b7 diff --git a/solr/licenses/jetty-alpn-server-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-alpn-server-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index af581055e77..00000000000 --- a/solr/licenses/jetty-alpn-server-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5dd67dacaf1eed80ab95493da840dab35c22ce9c diff --git a/solr/licenses/jetty-alpn-server-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-alpn-server-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..4ff1fdaff6d --- /dev/null +++ b/solr/licenses/jetty-alpn-server-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +aaacd77f8073e98f8400d042e70538623b3924ed diff --git a/solr/licenses/jetty-client-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-client-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 7b4cb62858c..00000000000 --- a/solr/licenses/jetty-client-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4386c1f243042e0f78f2e4c3c6cd239967410d6e diff --git a/solr/licenses/jetty-client-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-client-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..56d010b318f --- /dev/null +++ b/solr/licenses/jetty-client-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +09dd286abe644513305864453030e9c1631b5535 diff --git a/solr/licenses/jetty-continuation-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-continuation-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index d32802987d7..00000000000 --- a/solr/licenses/jetty-continuation-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4acddfa41f45790e43fe4be257c3c4bcf6b846ff diff --git a/solr/licenses/jetty-continuation-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-continuation-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..54d85735811 --- /dev/null +++ b/solr/licenses/jetty-continuation-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +69125cf74b07f1b9d60b5c94da47cb04c098f654 diff --git a/solr/licenses/jetty-deploy-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-deploy-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 6bf5df31291..00000000000 --- a/solr/licenses/jetty-deploy-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8b350466ff1fcb7030a7abc152eed458e086fac2 diff --git a/solr/licenses/jetty-deploy-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-deploy-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..16ad0708d87 --- /dev/null +++ b/solr/licenses/jetty-deploy-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +12d71fe6d671c635f1ae6fd3c31bc6578a293c4b diff --git a/solr/licenses/jetty-http-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-http-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 76f1249a504..00000000000 --- a/solr/licenses/jetty-http-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b59ff8ecb0cf5d6234958f2404eabf0b72464e14 diff --git a/solr/licenses/jetty-http-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-http-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..93c2d9afc19 --- /dev/null +++ b/solr/licenses/jetty-http-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +d3f0b0fb016ef8d35ffb199d928ffbcbfa121c86 diff --git a/solr/licenses/jetty-io-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-io-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 5af4939f6a3..00000000000 --- a/solr/licenses/jetty-io-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7eb9a6be62d84e1691e5fdc99223e632485619a8 diff --git a/solr/licenses/jetty-io-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-io-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..1579180a83c --- /dev/null +++ b/solr/licenses/jetty-io-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +dcb6d4d505ef74898e3a64a38c40195c01e97119 diff --git a/solr/licenses/jetty-jmx-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-jmx-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 960d08a24cb..00000000000 --- a/solr/licenses/jetty-jmx-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8dc81acdc4d3085c0b5f3c80b9a78cc9cb48bc4e diff --git a/solr/licenses/jetty-jmx-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-jmx-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..e148954c416 --- /dev/null +++ b/solr/licenses/jetty-jmx-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +22be18a055850a6cf3b0efd56c789c3929c87e98 diff --git a/solr/licenses/jetty-rewrite-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-rewrite-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 76f5e9e36cd..00000000000 --- a/solr/licenses/jetty-rewrite-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9b830886bd6098c613ed08d99574bbf300519506 diff --git a/solr/licenses/jetty-rewrite-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-rewrite-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..a573fa6c14a --- /dev/null +++ b/solr/licenses/jetty-rewrite-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +7990b0f4e8cafe99b47148df9aa3276af336d4d5 diff --git a/solr/licenses/jetty-security-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-security-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index c873d9a299f..00000000000 --- a/solr/licenses/jetty-security-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bfe96e1e78719bdd446e063c3f45c132010237ce diff --git a/solr/licenses/jetty-security-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-security-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..e702f11b383 --- /dev/null +++ b/solr/licenses/jetty-security-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +9fa640d36c088cf55843900043d28aef830ade4d diff --git a/solr/licenses/jetty-server-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-server-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index cbfdf11ae38..00000000000 --- a/solr/licenses/jetty-server-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -55786f6e6649bd49425a7da1ac72cd85b8dd4bef diff --git a/solr/licenses/jetty-server-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-server-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..ffd2c58fd5c --- /dev/null +++ b/solr/licenses/jetty-server-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +7885cc3d5d7701a444acada7ab97f89846514875 diff --git a/solr/licenses/jetty-servlet-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-servlet-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index c36dab5c770..00000000000 --- a/solr/licenses/jetty-servlet-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -89e25610b3199fdf34a831c1b306f7e765928959 diff --git a/solr/licenses/jetty-servlet-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-servlet-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..3ad349e3c0a --- /dev/null +++ b/solr/licenses/jetty-servlet-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +ca1803fde51b795c0a8346ca8bc6277d9d04d01d diff --git a/solr/licenses/jetty-servlets-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-servlets-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index c17d5463a4f..00000000000 --- a/solr/licenses/jetty-servlets-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b290c176abe2cd9274b9f794bf74497c4759359d diff --git a/solr/licenses/jetty-servlets-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-servlets-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..27c5b1a751d --- /dev/null +++ b/solr/licenses/jetty-servlets-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +b7bb7913d7583ee8d877f1c20feeb0905f342ad5 diff --git a/solr/licenses/jetty-util-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-util-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 0a2b5ade4f3..00000000000 --- a/solr/licenses/jetty-util-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2fd3cd40279280e8c56241f753d2c52d8d446d19 diff --git a/solr/licenses/jetty-util-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-util-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..a02b48126d5 --- /dev/null +++ b/solr/licenses/jetty-util-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +3095acb088f4ff9e3fd9aedf98db73e3c18ea849 diff --git a/solr/licenses/jetty-webapp-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-webapp-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index 60d1c20b072..00000000000 --- a/solr/licenses/jetty-webapp-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -945fc0c0fa69504c194e32c5330afa1df0be9574 diff --git a/solr/licenses/jetty-webapp-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-webapp-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..73263fa2f84 --- /dev/null +++ b/solr/licenses/jetty-webapp-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +968d70676fa16b3d62487987624dd4e9ce5db123 diff --git a/solr/licenses/jetty-xml-9.4.19.v20190610.jar.sha1 b/solr/licenses/jetty-xml-9.4.19.v20190610.jar.sha1 deleted file mode 100644 index bcfa2664346..00000000000 --- a/solr/licenses/jetty-xml-9.4.19.v20190610.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d25e67fbe0809cae777065b75b10ecfb5c1bd749 diff --git a/solr/licenses/jetty-xml-9.4.24.v20191120.jar.sha1 b/solr/licenses/jetty-xml-9.4.24.v20191120.jar.sha1 new file mode 100644 index 00000000000..628855bc514 --- /dev/null +++ b/solr/licenses/jetty-xml-9.4.24.v20191120.jar.sha1 @@ -0,0 +1 @@ +a5da43f9b72d2208b8f4d22ba0a16d176b328b91 diff --git a/solr/licenses/metrics-core-4.0.5.jar.sha1 b/solr/licenses/metrics-core-4.0.5.jar.sha1 deleted file mode 100644 index f5e12e83aa1..00000000000 --- a/solr/licenses/metrics-core-4.0.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b81ef162970cdb9f4512ee2da09715a856ff4c4c diff --git a/solr/licenses/metrics-core-4.1.2.jar.sha1 b/solr/licenses/metrics-core-4.1.2.jar.sha1 new file mode 100644 index 00000000000..8336aed7076 --- /dev/null +++ b/solr/licenses/metrics-core-4.1.2.jar.sha1 @@ -0,0 +1 @@ +bba231bbf3024c19e75622ec168821cbbd4261a4 diff --git a/solr/licenses/metrics-graphite-4.0.5.jar.sha1 b/solr/licenses/metrics-graphite-4.0.5.jar.sha1 deleted file mode 100644 index 2aa9abb956e..00000000000 --- a/solr/licenses/metrics-graphite-4.0.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -76e8758356373d5aed5abacbda429b38f6e8fa98 diff --git a/solr/licenses/metrics-graphite-4.1.2.jar.sha1 b/solr/licenses/metrics-graphite-4.1.2.jar.sha1 new file mode 100644 index 00000000000..548e550854e --- /dev/null +++ b/solr/licenses/metrics-graphite-4.1.2.jar.sha1 @@ -0,0 +1 @@ +c98a2821eeb9193001c131a6d742a8f4e67e3b10 diff --git a/solr/licenses/metrics-jetty9-4.0.5.jar.sha1 b/solr/licenses/metrics-jetty9-4.0.5.jar.sha1 deleted file mode 100644 index 228b179faa1..00000000000 --- a/solr/licenses/metrics-jetty9-4.0.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -87f3b49a7377e56f62046875d394ed0028b37690 diff --git a/solr/licenses/metrics-jetty9-4.1.2.jar.sha1 b/solr/licenses/metrics-jetty9-4.1.2.jar.sha1 new file mode 100644 index 00000000000..57ec4f0dc18 --- /dev/null +++ b/solr/licenses/metrics-jetty9-4.1.2.jar.sha1 @@ -0,0 +1 @@ +96b3aefcd0544de7e0e1f72990968b48c3d04dd1 diff --git a/solr/licenses/metrics-jmx-4.0.5.jar.sha1 b/solr/licenses/metrics-jmx-4.0.5.jar.sha1 deleted file mode 100644 index b70a07de05d..00000000000 --- a/solr/licenses/metrics-jmx-4.0.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d7be4ddd7ba674ee8be1d23d883fb3ca68ee1d54 diff --git a/solr/licenses/metrics-jmx-4.1.2.jar.sha1 b/solr/licenses/metrics-jmx-4.1.2.jar.sha1 new file mode 100644 index 00000000000..63d4d328b86 --- /dev/null +++ b/solr/licenses/metrics-jmx-4.1.2.jar.sha1 @@ -0,0 +1 @@ +73a242395eadde5bc4cd16a43230531dccb3d0eb diff --git a/solr/licenses/metrics-jvm-4.0.5.jar.sha1 b/solr/licenses/metrics-jvm-4.0.5.jar.sha1 deleted file mode 100644 index 176a65f9ff1..00000000000 --- a/solr/licenses/metrics-jvm-4.0.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -09f6f1e6c1db440d9ad4c3114f17be40f66bb399 diff --git a/solr/licenses/metrics-jvm-4.1.2.jar.sha1 b/solr/licenses/metrics-jvm-4.1.2.jar.sha1 new file mode 100644 index 00000000000..2c0813b7b7f --- /dev/null +++ b/solr/licenses/metrics-jvm-4.1.2.jar.sha1 @@ -0,0 +1 @@ +01c23f7c463f8f4e83209d83b0549cea5d51ec1c diff --git a/solr/licenses/start.jar.sha1 b/solr/licenses/start.jar.sha1 index e7dd6a27ae2..93a18845755 100644 --- a/solr/licenses/start.jar.sha1 +++ b/solr/licenses/start.jar.sha1 @@ -1 +1 @@ -6ab950be264e74803f12ba43ee5db434a74e6c0c +ddb54190e858875fb681a6b9dd630a3609eaa513 From 2feeb88c294030be654183d22a8e503bbbf8b293 Mon Sep 17 00:00:00 2001 From: erick Date: Fri, 13 Dec 2019 12:42:09 -0500 Subject: [PATCH 02/15] SOLR-14026: Upgrade Jetty to 9.4.24.v20191120 and dropwizard to 4.1.2, moved to 8.5 in CHANGES.txt --- solr/CHANGES.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 5811218f5c4..d03fe064213 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -26,7 +26,7 @@ Apache Tika 1.19.1 Carrot2 3.16.2 Velocity 2.0 and Velocity Tools 3.0 Apache ZooKeeper 3.5.5 -Jetty 9.4.19.v20190610 +Jetty 9.4.24.v20191120 Upgrade Notes ---------------------- @@ -105,7 +105,8 @@ Jetty 9.4.24.v20191120 Upgrade Notes --------------------- -(No changes) + +* SOLR-14026: Upgrade Jetty to 9.4.24.v20191120 and dropwizard to 4.1.2 (Erick Erickson) New Features --------------------- @@ -179,8 +180,6 @@ Upgrade Notes * SOLR-14065: VelocityResponseWriter has been deprecated and may be removed in a future version. -* SOLR-14026: Upgrade Jetty to 9.4.24.v20191120 and dropwizard to 4.1.2 (Erick Erickson) - New Features --------------------- * SOLR-13821: A Package store to store and load package artifacts (noble, Ishan Chattopadhyaya) From fae6aebc74291280c8154e15e505de8bdbe0a349 Mon Sep 17 00:00:00 2001 From: yonik Date: Wed, 30 Oct 2019 15:23:55 -0400 Subject: [PATCH 03/15] SOLR-13884: add ConcurrentCreateCollectionTest test --- .../ConcurrentCreateCollectionTest.java | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java new file mode 100644 index 00000000000..042e14d605b --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud.api.collections; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.lucene.util.LuceneTestCase.Nightly; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.cloud.MiniSolrCloudCluster; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.util.IOUtils; +import org.apache.solr.common.util.TimeSource; +import org.apache.solr.util.TimeOut; +import org.apache.zookeeper.KeeperException; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(2) + .addConfig("conf", configset("cloud-minimal")) + .configure(); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + cluster.deleteAllCollections(); + } + + + + public void testConcurrentCreatePlacement() throws Exception { + final int nThreads = 20; + final int createsPerThread = 1; + final int repFactor = 1; + + final CloudSolrClient client = cluster.getSolrClient(); + + final AtomicInteger collectionNum = new AtomicInteger(); + Thread[] indexThreads = new Thread[nThreads]; + + + for (int i=0; i { + try { + for (int j=0; j> map = new HashMap<>(); + ClusterState cstate = client.getZkStateReader().getClusterState(); + for (DocCollection collection : cstate.getCollectionsMap().values()) { + for (Replica replica : collection.getReplicas()) { + String url = replica.getBaseUrl(); + List replicas = map.get(url); + if (replicas == null) { + replicas = new ArrayList<>(); + map.put(url, replicas); + } + replicas.add(replica); + } + } + + // check if nodes are balanced + List prev = null; + for (List replicas : map.values()) { + if (prev != null && prev.size() != replicas.size()) { + log.error("UNBALANCED CLUSTER: prev node replica count=" + prev.size() + " current=" + replicas.size() + "\n" + cstate.getCollectionsMap()); + log.error("Replica lists per node: " + map); + assertEquals(prev.size(), replicas.size()); + } + prev = replicas; + } + + } + + + +} From 73c535261c53d61742516e633edea0036b0f3bf0 Mon Sep 17 00:00:00 2001 From: yonik Date: Thu, 31 Oct 2019 13:13:22 -0400 Subject: [PATCH 04/15] SOLR-13884: use policies, preferences --- .../ConcurrentCreateCollectionTest.java | 78 ++++++++++++++----- 1 file changed, 59 insertions(+), 19 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java index 042e14d605b..8842b3c48f7 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java @@ -33,12 +33,14 @@ import org.apache.lucene.util.LuceneTestCase.Nightly; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.cloud.CloudTestUtils; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrDocument; @@ -60,10 +62,13 @@ public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + static int NODES = 2; + @BeforeClass public static void setupCluster() throws Exception { - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) + configureCluster(NODES) + // .addConfig("conf", configset("cloud-minimal")) + .addConfig("conf", configset("_default")) .configure(); } @@ -86,25 +91,52 @@ public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { final int nThreads = 20; final int createsPerThread = 1; final int repFactor = 1; + final boolean useClusterPolicy = true; + final boolean useCollectionPolicy = false; final CloudSolrClient client = cluster.getSolrClient(); + + if (useClusterPolicy) { + String setClusterPolicyCommand = "{" + + " 'set-cluster-policy': [" + + // " {'cores':'<100', 'node':'#ANY'}," + + " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + + " ]" + + "}"; + + SolrRequest req = CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); + client.request(req); + } + + if (useCollectionPolicy) { + // NOTE: the meer act of setting this named policy prevents LegacyAssignStrategy from being used, even if the policy is + // not used during collection creation. + String commands = "{set-policy :{policy1 : [{replica:'<2' , node:'#ANY'}]}}"; + client.request(CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); + } + + final AtomicInteger collectionNum = new AtomicInteger(); Thread[] indexThreads = new Thread[nThreads]; - for (int i=0; i { try { for (int j=0; j> map = new HashMap<>(); + int expectedTotalReplicas = nThreads * createsPerThread * repFactor; + int expectedPerNode = expectedTotalReplicas / NODES; + + Map> replicaMap = new HashMap<>(); ClusterState cstate = client.getZkStateReader().getClusterState(); for (DocCollection collection : cstate.getCollectionsMap().values()) { for (Replica replica : collection.getReplicas()) { String url = replica.getBaseUrl(); - List replicas = map.get(url); + List replicas = replicaMap.get(url); if (replicas == null) { replicas = new ArrayList<>(); - map.put(url, replicas); + replicaMap.put(url, replicas); } replicas.add(replica); } } // check if nodes are balanced - List prev = null; - for (List replicas : map.values()) { - if (prev != null && prev.size() != replicas.size()) { - log.error("UNBALANCED CLUSTER: prev node replica count=" + prev.size() + " current=" + replicas.size() + "\n" + cstate.getCollectionsMap()); - log.error("Replica lists per node: " + map); - assertEquals(prev.size(), replicas.size()); + boolean failed = false; + for (List replicas : replicaMap.values()) { + if (replicas.size() != expectedPerNode ) { + failed = true; + log.error("UNBALANCED CLUSTER: expected replicas per node " + expectedPerNode + " but got " + replicas.size()); } - prev = replicas; } + if (failed) { + log.error("Cluster state " + cstate.getCollectionsMap()); + } + + assertEquals(replicaMap.size(), NODES); // make sure something was created + + assertTrue(!failed); } From db65c82c3914accd8117701b5d4c164a5b3cafc5 Mon Sep 17 00:00:00 2001 From: yonik Date: Mon, 4 Nov 2019 11:24:36 -0500 Subject: [PATCH 05/15] SOLR-13884: detect multiple replicas on single node --- .../ConcurrentCreateCollectionTest.java | 160 ++++++++++++++---- 1 file changed, 130 insertions(+), 30 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java index 8842b3c48f7..6d103f14165 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java @@ -16,48 +16,31 @@ */ package org.apache.solr.cloud.api.collections; -import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import org.apache.lucene.util.LuceneTestCase.Nightly; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.util.IOUtils; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.util.TimeOut; -import org.apache.zookeeper.KeeperException; +import org.apache.solr.common.cloud.Slice; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + + public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -86,22 +69,94 @@ public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { } + private CollectionAdminRequest.Create createCollectionRequest(String cname, int numShards, int numReplicas) throws Exception { + CollectionAdminRequest.Create creq = CollectionAdminRequest + // nocommit .createCollection(cname, "conf", NODES - 1, NODES - 1) + .createCollection(cname, "conf", numShards, numReplicas) + .setMaxShardsPerNode(100); + creq.setWaitForFinalState(true); + creq.setAutoAddReplicas(true); + return creq; + } public void testConcurrentCreatePlacement() throws Exception { - final int nThreads = 20; + final int nThreads = 2; final int createsPerThread = 1; - final int repFactor = 1; - final boolean useClusterPolicy = true; - final boolean useCollectionPolicy = false; + final int nShards = 1; + final int repFactor = 2; + final boolean useClusterPolicy = false; + final boolean useCollectionPolicy = true; + final boolean startUnbalanced = true; // can help make a smaller test that can still reproduce an issue. + final int unbalancedSize = 1; // the number of replicas to create first + final boolean stopNode = false; // only applicable when startUnbalanced==true... stops a node during first collection creation, then restarts final CloudSolrClient client = cluster.getSolrClient(); + if (startUnbalanced) { + /*** This produces a failure (multiple replicas of single shard on same node) when run with NODES=4 and + final int nThreads = 2; + final int createsPerThread = 1; + final int nShards = 2; + final int repFactor = 2; + final boolean useClusterPolicy = false; + final boolean useCollectionPolicy = true; + final boolean startUnbalanced = true; + // NOTE: useClusterPolicy=true seems to fix it! So does putting both creates in a single thread! + // NOTE: even creating a single replica to start with causes failure later on. + + Also reproduced with smaller cluster: NODES=2 and + final int nThreads = 2; + final int createsPerThread = 1; + final int nShards = 1; + final int repFactor = 2; + final boolean useClusterPolicy = false; + final boolean useCollectionPolicy = true; + final boolean startUnbalanced = true; + + Also, with NODES=3: + final int nThreads = 2; + final int createsPerThread = 1; + final int nShards = 1; + final int repFactor = 2; + final boolean useClusterPolicy = false; + final boolean useCollectionPolicy = true; + final boolean startUnbalanced = false; + + // Also succeeded in replicating a bug where all 5 replicas were on a single node: CORES=5, nThreads=5, repFactor=5, + // unbalancedSize = 16 (4 replicas on each of the up nodes), stopNode=true + ***/ + + + JettySolrRunner downJetty = cluster.getJettySolrRunners().get(0); + if (stopNode) { + cluster.stopJettySolrRunner(downJetty); + } + + String cname = "STARTCOLLECTION"; + CollectionAdminRequest.Create creq = CollectionAdminRequest + // nocommit .createCollection(cname, "conf", NODES - 1, NODES - 1) + .createCollection(cname, "conf", unbalancedSize, 1) + .setMaxShardsPerNode(100); + creq.setWaitForFinalState(true); + // creq.setAutoAddReplicas(true); + if (useCollectionPolicy) { creq.setPolicy("policy1"); } + creq.process(client); + + if (stopNode) { + // this will start it with a new port.... does it matter? + cluster.startJettySolrRunner(downJetty); + } + } + + + if (useClusterPolicy) { String setClusterPolicyCommand = "{" + " 'set-cluster-policy': [" + // " {'cores':'<100', 'node':'#ANY'}," + " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + + // " {'replica':'<2', 'node': '#ANY'}," + " ]" + "}"; @@ -110,12 +165,36 @@ public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { } if (useCollectionPolicy) { - // NOTE: the meer act of setting this named policy prevents LegacyAssignStrategy from being used, even if the policy is + // NOTE: the mere act of setting this named policy prevents LegacyAssignStrategy from being used, even if the policy is // not used during collection creation. - String commands = "{set-policy :{policy1 : [{replica:'<2' , node:'#ANY'}]}}"; + String commands = "{set-policy : {" + + " policy1 : [{replica:'<2' , node:'#ANY'}]" + + ",policy2 : [{replica:'<2' , shard:'#EACH', node:'#ANY'}]" + + "}}"; client.request(CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); + + /*** take defaults for cluster preferences + String cmd = "{" + + " 'set-cluster-preferences': [" + + // " {'cores':'<100', 'node':'#ANY'}," + + " {minimize:cores}" + + " ]" + + "}"; + + SolrRequest req = CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.POST, cmd); + client.request(req); + ***/ } + /*** + SolrRequest req = CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.GET, null); + SolrResponse response = req.process(client); + log.info("######### AUTOSCALE " + response); + ***/ + + + byte[] data = client.getZkStateReader().getZkClient().getData("/autoscaling.json", null, null, true); + log.info("AUTOSCALE DATA: " + new String(data, "UTF-8")); final AtomicInteger collectionNum = new AtomicInteger(); Thread[] indexThreads = new Thread[nThreads]; @@ -125,14 +204,17 @@ public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { try { for (int j=0; j> replicaMap = new HashMap<>(); ClusterState cstate = client.getZkStateReader().getClusterState(); @@ -173,11 +256,28 @@ public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { boolean failed = false; for (List replicas : replicaMap.values()) { if (replicas.size() != expectedPerNode ) { - failed = true; + if (expectBalanced) { + failed = true; + } log.error("UNBALANCED CLUSTER: expected replicas per node " + expectedPerNode + " but got " + replicas.size()); } } + // check if there were multiple replicas of the same shard placed on the same node + for (DocCollection collection : cstate.getCollectionsMap().values()) { + for (Slice slice : collection.getSlices()) { + Map nodeToReplica = new HashMap<>(); + for (Replica replica : slice.getReplicas()) { + Replica prev = nodeToReplica.put(replica.getBaseUrl(), replica); + if (prev != null) { + failed = true; + // NOTE: with a replication factor > 2, this will print multiple times per bad slice. + log.error("MULTIPLE REPLICAS OF SINGLE SHARD ON SAME NODE: r1=" + prev + " r2=" + replica); + } + } + } + } + if (failed) { log.error("Cluster state " + cstate.getCollectionsMap()); } From 5f8e65c58f5849302d3aaacd65eea46eab258a57 Mon Sep 17 00:00:00 2001 From: yonik Date: Fri, 13 Dec 2019 17:55:05 -0500 Subject: [PATCH 06/15] SOLR-14079: fix SPLITSHARD splitByPrefix in async mode --- solr/CHANGES.txt | 4 ++++ .../cloud/api/collections/SplitShardCmd.java | 3 +-- .../api/collections/SplitByPrefixTest.java | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d03fe064213..05ddf486139 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -315,6 +315,10 @@ Bug Fixes * SOLR-14013: FIX: javabin performance regressions (noble, yonik, Houston Putman) +* SOLR-14079: SPLITSHARD splitByPrefix doesn't work in async mode. This also + affects splits triggered by the autoscale framework, which use async mode. + (Megan Carey, Andy Vuong, Bilal Waheed, Ilan Ginzburg, yonik) + Other Changes --------------------- diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java index 333051a654e..26b1f0d23ad 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java @@ -223,12 +223,11 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd { // params.set(NUM_SUB_SHARDS, Integer.toString(numSubShards)); { - final ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId); + final ShardRequestTracker shardRequestTracker = ocmh.syncRequestTracker(); shardRequestTracker.sendShardRequest(parentShardLeader.getNodeName(), params, shardHandler); SimpleOrderedMap getRangesResults = new SimpleOrderedMap<>(); String msgOnError = "SPLITSHARD failed to invoke SPLIT.getRanges core admin command"; shardRequestTracker.processResponses(getRangesResults, shardHandler, true, msgOnError); - handleFailureOnAsyncRequest(results, msgOnError); // Extract the recommended splits from the shard response (if it exists) // example response: getRangesResults={success={127.0.0.1:62086_solr={responseHeader={status=0,QTime=1},ranges=10-20,3a-3f}}} diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/SplitByPrefixTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/SplitByPrefixTest.java index 5ec53e5cbbf..cca4fe462ae 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/SplitByPrefixTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/SplitByPrefixTest.java @@ -148,6 +148,10 @@ public class SplitByPrefixTest extends SolrCloudTestCase { @Test public void doTest() throws IOException, SolrServerException { + // SPLITSHARD is recommended to be run in async mode, so we default to that. + // Also, autoscale triggers use async with splits as well. + boolean doAsync = true; + CollectionAdminRequest .createCollection(COLLECTION_NAME, "conf", 1, 1) .setMaxShardsPerNode(100) @@ -165,6 +169,9 @@ public class SplitByPrefixTest extends SolrCloudTestCase { .setNumSubShards(2) .setSplitByPrefix(true) .setShardName("shard1"); + if (doAsync) { + splitShard.setAsyncId("SPLIT1"); + } splitShard.process(client); waitForState("Timed out waiting for sub shards to be active.", COLLECTION_NAME, activeClusterShape(2, 3)); // expectedReplicas==3 because original replica still exists (just inactive) @@ -187,6 +194,9 @@ public class SplitByPrefixTest extends SolrCloudTestCase { splitShard = CollectionAdminRequest.splitShard(COLLECTION_NAME) .setSplitByPrefix(true) .setShardName("shard1_1"); // should start out with the range of 0-7fffffff + if (doAsync) { + splitShard.setAsyncId("SPLIT2"); + } splitShard.process(client); waitForState("Timed out waiting for sub shards to be active.", COLLECTION_NAME, activeClusterShape(3, 5)); @@ -216,6 +226,9 @@ public class SplitByPrefixTest extends SolrCloudTestCase { splitShard = CollectionAdminRequest.splitShard(COLLECTION_NAME) .setSplitByPrefix(true) .setShardName(slice1.getName()); + if (doAsync) { + splitShard.setAsyncId("SPLIT3"); + } splitShard.process(client); waitForState("Timed out waiting for sub shards to be active.", COLLECTION_NAME, activeClusterShape(4, 7)); @@ -236,6 +249,9 @@ public class SplitByPrefixTest extends SolrCloudTestCase { splitShard = CollectionAdminRequest.splitShard(COLLECTION_NAME) .setSplitByPrefix(true) .setShardName(slice1.getName()); + if (doAsync) { + splitShard.setAsyncId("SPLIT4"); + } splitShard.process(client); waitForState("Timed out waiting for sub shards to be active.", COLLECTION_NAME, activeClusterShape(5, 9)); @@ -252,6 +268,9 @@ public class SplitByPrefixTest extends SolrCloudTestCase { splitShard = CollectionAdminRequest.splitShard(COLLECTION_NAME) .setSplitByPrefix(true) .setShardName(slice1.getName()); + if (doAsync) { + splitShard.setAsyncId("SPLIT5"); + } splitShard.process(client); waitForState("Timed out waiting for sub shards to be active.", COLLECTION_NAME, activeClusterShape(6, 11)); From 7a9a6ef79e9eee98d5f20ab7256f01a20113e953 Mon Sep 17 00:00:00 2001 From: Kevin Risden Date: Fri, 13 Dec 2019 16:39:47 -0500 Subject: [PATCH 07/15] SOLR-14077: Hadoop shouldn't need to look for metrics config in user home Signed-off-by: Kevin Risden --- lucene/tools/junit4/solr-tests.policy | 3 --- .../org/apache/solr/cloud/hdfs/HdfsTestUtil.java | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lucene/tools/junit4/solr-tests.policy b/lucene/tools/junit4/solr-tests.policy index faa9ecb2317..b178d6be6c5 100644 --- a/lucene/tools/junit4/solr-tests.policy +++ b/lucene/tools/junit4/solr-tests.policy @@ -33,9 +33,6 @@ grant { permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete"; permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete"; permission java.io.FilePermission "${tests.linedocsfile}", "read"; - // hadoop - permission java.io.FilePermission "${user.home}${/}hadoop-metrics2.properties", "read"; - permission java.io.FilePermission "${user.home}${/}hadoop-metrics2-namenode.properties", "read"; // DirectoryFactoryTest messes with these (wtf?) permission java.io.FilePermission "/tmp/inst1/conf/solrcore.properties", "read"; permission java.io.FilePermission "/path/to/myinst/conf/solrcore.properties", "read"; diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java index 98f82747a92..873fe17d747 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java @@ -47,6 +47,9 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeResourceChecker; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.util.DiskChecker; import org.apache.lucene.util.Constants; import org.apache.lucene.util.LuceneTestCase; @@ -159,6 +162,8 @@ public class HdfsTestUtil { if (!HA_TESTING_ENABLED) haTesting = false; + DefaultMetricsSystem.setInstance(new FakeMetricsSystem()); + Configuration conf = getBasicConfiguration(new Configuration()); conf.set("hdfs.minidfs.basedir", dir + File.separator + "hdfsBaseDir"); conf.set("dfs.namenode.name.dir", dir + File.separator + "nameNodeNameDir"); @@ -374,4 +379,15 @@ public class HdfsTestUtil { super(pool); } } + + /** + * Ensures that we don't try to initialize metrics and read files outside + * the source tree. + */ + public static class FakeMetricsSystem extends MetricsSystemImpl { + @Override + public synchronized MetricsSystem init(String prefix) { + return this; + } + } } From 49c34028ab6b840f6582f4397a28f1a9e11101cb Mon Sep 17 00:00:00 2001 From: yonik Date: Sat, 14 Dec 2019 09:31:13 -0500 Subject: [PATCH 08/15] SOLR-13884: Ignore ConcurrentCreateCollectionTest --- .../api/collections/ConcurrentCreateCollectionTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java index 6d103f14165..b864ce28a23 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java @@ -36,11 +36,12 @@ import org.apache.solr.common.cloud.Slice; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Ignore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +@Ignore("SOLR-13884") public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -71,7 +72,7 @@ public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { private CollectionAdminRequest.Create createCollectionRequest(String cname, int numShards, int numReplicas) throws Exception { CollectionAdminRequest.Create creq = CollectionAdminRequest - // nocommit .createCollection(cname, "conf", NODES - 1, NODES - 1) + // .createCollection(cname, "conf", NODES - 1, NODES - 1) .createCollection(cname, "conf", numShards, numReplicas) .setMaxShardsPerNode(100); creq.setWaitForFinalState(true); @@ -135,7 +136,7 @@ public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { String cname = "STARTCOLLECTION"; CollectionAdminRequest.Create creq = CollectionAdminRequest - // nocommit .createCollection(cname, "conf", NODES - 1, NODES - 1) + // .createCollection(cname, "conf", NODES - 1, NODES - 1) .createCollection(cname, "conf", unbalancedSize, 1) .setMaxShardsPerNode(100); creq.setWaitForFinalState(true); @@ -204,7 +205,7 @@ public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { try { for (int j=0; j Date: Sat, 14 Dec 2019 11:50:00 -0500 Subject: [PATCH 09/15] SOLR-14069: Ref guide: overhaul: resources, libs, plugins, config-sets (#1077) * split "resource-and-plugin-loading.adoc" into "resource-loading.adoc" and "libs.adoc" then overhauled both. * enhanced "config-sets.adoc", moving some content in from elsewhere; bit of an overhaul. * solr-plugins.adoc is now top-level; overhauled content * Move resource-loading.adoc up a level in the TOC to underneath "The Well-Configured Solr Instance. * Separate out the leading sentence. --- solr/solr-ref-guide/src/analytics.adoc | 7 +- solr/solr-ref-guide/src/config-sets.adoc | 29 +++++-- .../src/configuring-solrconfig-xml.adoc | 16 +++- .../detecting-languages-during-indexing.adoc | 2 +- .../src/filter-descriptions.adoc | 8 +- solr/solr-ref-guide/src/index.adoc | 21 ++++- .../solr-ref-guide/src/language-analysis.adoc | 32 +++---- solr/solr-ref-guide/src/learning-to-rank.adoc | 2 +- solr/solr-ref-guide/src/libs.adoc | 78 +++++++++++++++++ .../src/resource-and-plugin-loading.adoc | 86 ------------------- solr/solr-ref-guide/src/resource-loading.adoc | 44 ++++++++++ solr/solr-ref-guide/src/solr-plugins.adoc | 40 ++++++++- .../the-well-configured-solr-instance.adoc | 15 ++-- solr/solr-ref-guide/src/tokenizers.adoc | 2 +- .../src/update-request-processors.adoc | 2 +- 15 files changed, 252 insertions(+), 132 deletions(-) create mode 100644 solr/solr-ref-guide/src/libs.adoc delete mode 100644 solr/solr-ref-guide/src/resource-and-plugin-loading.adoc create mode 100644 solr/solr-ref-guide/src/resource-loading.adoc diff --git a/solr/solr-ref-guide/src/analytics.adoc b/solr/solr-ref-guide/src/analytics.adoc index 748ee88fe6b..3ad70670a4f 100644 --- a/solr/solr-ref-guide/src/analytics.adoc +++ b/solr/solr-ref-guide/src/analytics.adoc @@ -33,12 +33,9 @@ Since the Analytics framework is a _search component_, it must be declared as su For distributed analytics requests over cloud collections, the component uses the `AnalyticsHandler` strictly for inter-shard communication. The Analytics Handler should not be used by users to submit analytics requests. -To configure Solr to use the Analytics Component, the first step is to add a `` directive so Solr loads the Analytic Component classes (for more about the `` directive, see <>). In the section of `solrconfig.xml` where the default `` directives are, add a line: +To use the Analytics Component, the first step is to install this contrib module's plugins into Solr -- see the <> section on how to do this. -[source,xml] - - -Next you need to enable the request handler and search component. Add the following lines to `solrconfig.xml`, near the defintions for other request handlers: +Next you need to register the request handler and search component. Add the following lines to `solrconfig.xml`, near the defintions for other request handlers: [source,xml] .solrconfig.xml diff --git a/solr/solr-ref-guide/src/config-sets.adoc b/solr/solr-ref-guide/src/config-sets.adoc index 7bb8a25035c..b846dcbbcd4 100644 --- a/solr/solr-ref-guide/src/config-sets.adoc +++ b/solr/solr-ref-guide/src/config-sets.adoc @@ -16,17 +16,24 @@ // specific language governing permissions and limitations // under the License. -On a multicore Solr instance, you may find that you want to share configuration between a number of different cores. You can achieve this using named configsets, which are essentially shared configuration directories stored under a configurable configset base directory. +Configsets are a set of configuration files used in a Solr installation: `solrconfig.xml`, the schema, and then <> like language files, `synonyms.txt`, DIH-related configuration, and others that are referenced from the config or schema. -Configsets are made up of the configuration files used in a Solr installation: inclduding `solrconfig.xml`, the schema, language-files, `synonyms.txt`, DIH-related configuration, and others as needed for your implementation. +Such configuration, _configsets_, can be named and then referenced by collections or cores, possibly with the intent to share them to avoid duplication. Solr ships with two example configsets located in `server/solr/configsets`, which can be used as a base for your own. These example configsets are named `_default` and `sample_techproducts_configs`. == Configsets in Standalone Mode -If you are using Solr in standalone mode, configsets are created on the filesystem. +If you are using Solr in standalone mode, configsets are managed on the filesystem. -To create a configset, add a new directory under the configset base directory. The configset will be identified by the name of this directory. Then into this copy the configuration directory you want to share. The structure should look something like this: +Each Solr core can have it's very own configSet located beneath it in a `/conf/` dir. +Here, it is not named or shared and the word _configset_ isn't found. +In Solr's early years, this was _the only way_ it was configured. + +To create a named configset, add a new directory under the configset base directory. +The configset will be identified by the name of this directory. +Then add a `conf/` directory containing the configuration you want to share. +The structure should look something like this: [source,bash] ---- @@ -76,4 +83,16 @@ curl -v -X POST -H 'Content-type: application/json' -d '{ == Configsets in SolrCloud Mode -In SolrCloud mode, you can use the <> to manage your configsets. +In SolrCloud, it's critical to understand that configsets are fundamentally stored in ZooKeeper _and not_ the file system. +Solr's `_default` configset is uploaded to ZooKeeper on initialization. +This and some demonstration ones remain on the file system but Solr does not use them whatsoever in this mode. + +When you create a collection in SolrCloud, you can specify a named configset -- possibly shared. +If you don't, then the `_default` will be copied and given a unique name for use by this collection. + +A configset can be uploaded to ZooKeeper either via the <> or more directly via <>. +The Configsets API has some other operations as well, and likewise, so does the CLI. + +To upload a file to a configset already stored on ZooKeeper, you can use <>. + +CAUTION: By default, ZooKeeper's file size limit is 1MB. If your files are larger than this, you'll need to either <> or store them instead <>. \ No newline at end of file diff --git a/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc b/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc index aaeb31b3740..fccd9d27435 100644 --- a/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc +++ b/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc @@ -1,5 +1,15 @@ = Configuring solrconfig.xml -:page-children: datadir-and-directoryfactory-in-solrconfig, resource-and-plugin-loading, schema-factory-definition-in-solrconfig, indexconfig-in-solrconfig, requesthandlers-and-searchcomponents-in-solrconfig, initparams-in-solrconfig, updatehandlers-in-solrconfig, query-settings-in-solrconfig, requestdispatcher-in-solrconfig, update-request-processors, codec-factory +:page-children: datadir-and-directoryfactory-in-solrconfig, \ + schema-factory-definition-in-solrconfig, \ + indexconfig-in-solrconfig, \ + requesthandlers-and-searchcomponents-in-solrconfig, \ + initparams-in-solrconfig, \ + updatehandlers-in-solrconfig, \ + query-settings-in-solrconfig, \ + requestdispatcher-in-solrconfig, \ + update-request-processors, \ + codec-factory + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -38,7 +48,6 @@ The `solrconfig.xml` file is located in the `conf/` directory for each collectio We've covered the options in the following sections: * <> -* <> * <> * <> * <> @@ -49,6 +58,9 @@ We've covered the options in the following sections: * <> * <> +Some SolrConfig aspects are covered in other sections. +See <>, which can be used for both Plugins and Resources. + == Substituting Properties in Solr Config Files Solr supports variable substitution of property values in configuration files, which allows runtime specification of various configuration options in `solrconfig.xml`. The syntax is `${propertyname[:option default value]`}. This allows defining a default that can be overridden when Solr is launched. If a default value is not specified, then the property _must_ be specified at runtime or the configuration file will generate an error when parsed. diff --git a/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc b/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc index 8d446a29fe7..92e5986dead 100644 --- a/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc +++ b/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc @@ -80,7 +80,7 @@ Here is an example of a minimal OpenNLP `langid` configuration in `solrconfig.xm ==== OpenNLP-specific Parameters `langid.model`:: -An OpenNLP language detection model. The OpenNLP project provides a pre-trained 103 language model on the http://opennlp.apache.org/models.html[OpenNLP site's model dowload page]. Model training instructions are provided on the http://opennlp.apache.org/docs/{ivy-opennlp-version}/manual/opennlp.html#tools.langdetect[OpenNLP website]. This parameter is required. See <> for information on where to put the model. +An OpenNLP language detection model. The OpenNLP project provides a pre-trained 103 language model on the http://opennlp.apache.org/models.html[OpenNLP site's model dowload page]. Model training instructions are provided on the http://opennlp.apache.org/docs/{ivy-opennlp-version}/manual/opennlp.html#tools.langdetect[OpenNLP website]. This parameter is required. See <> for information on where to put the model. ==== OpenNLP Language Codes diff --git a/solr/solr-ref-guide/src/filter-descriptions.adoc b/solr/solr-ref-guide/src/filter-descriptions.adoc index f59a366a5b1..1ddfd531aa2 100644 --- a/solr/solr-ref-guide/src/filter-descriptions.adoc +++ b/solr/solr-ref-guide/src/filter-descriptions.adoc @@ -732,7 +732,7 @@ Note that for this filter to work properly, the upstream tokenizer must not remo This filter is a custom Unicode normalization form that applies the foldings specified in http://www.unicode.org/reports/tr30/tr30-4.html[Unicode TR #30: Character Foldings] in addition to the `NFKC_Casefold` normalization form as described in <>. This filter is a better substitute for the combined behavior of the <>, <>, and <>. -To use this filter, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. +To use this filter, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. *Factory class:* `solr.ICUFoldingFilterFactory` @@ -840,7 +840,7 @@ This filter factory normalizes text according to one of five Unicode Normalizati For detailed information about these normalization forms, see http://unicode.org/reports/tr15/[Unicode Normalization Forms]. -To use this filter, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. +To use this filter, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. == ICU Transform Filter @@ -882,7 +882,7 @@ This filter applies http://userguide.icu-project.org/transforms/general[ICU Tran For detailed information about ICU Transforms, see http://userguide.icu-project.org/transforms/general. -To use this filter, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. +To use this filter, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. == Keep Word Filter @@ -2210,7 +2210,7 @@ NOTE: Although this filter produces correct token graphs, it cannot consume an i *Arguments:* -`synonyms`:: (required) The path of a file that contains a list of synonyms, one per line. In the (default) `solr` format - see the `format` argument below for alternatives - blank lines and lines that begin with "`#`" are ignored. This may be a comma-separated list of paths. See <> for more information. +`synonyms`:: (required) The path of a file that contains a list of synonyms, one per line. In the (default) `solr` format - see the `format` argument below for alternatives - blank lines and lines that begin with "`#`" are ignored. This may be a comma-separated list of paths. See <> for more information. + There are two ways to specify synonym mappings: + diff --git a/solr/solr-ref-guide/src/index.adoc b/solr/solr-ref-guide/src/index.adoc index 53df5005aec..f187d2a7916 100644 --- a/solr/solr-ref-guide/src/index.adoc +++ b/solr/solr-ref-guide/src/index.adoc @@ -1,5 +1,24 @@ = Apache Solr Reference Guide -:page-children: about-this-guide, getting-started, deployment-and-operations, using-the-solr-administration-user-interface, documents-fields-and-schema-design, understanding-analyzers-tokenizers-and-filters, indexing-and-basic-data-operations, searching, streaming-expressions, solrcloud, legacy-scaling-and-distribution, the-well-configured-solr-instance, monitoring-solr, securing-solr, client-apis, further-assistance, solr-glossary, errata, how-to-contribute +:page-children: about-this-guide, \ + getting-started, \ + deployment-and-operations, \ + using-the-solr-administration-user-interface, \ + documents-fields-and-schema-design, \ + understanding-analyzers-tokenizers-and-filters, \ + indexing-and-basic-data-operations, \ + searching, \ + streaming-expressions, \ + solrcloud, \ + legacy-scaling-and-distribution, \ + solr-plugins, \ + the-well-configured-solr-instance, \ + monitoring-solr, \ + securing-solr, \ + client-apis, \ + further-assistance, \ + solr-glossary, \ + errata, \ + how-to-contribute :page-notitle: :page-toc: false :page-layout: home diff --git a/solr/solr-ref-guide/src/language-analysis.adoc b/solr/solr-ref-guide/src/language-analysis.adoc index d71b8f5ba8e..adeb4b006da 100644 --- a/solr/solr-ref-guide/src/language-analysis.adoc +++ b/solr/solr-ref-guide/src/language-analysis.adoc @@ -166,7 +166,7 @@ Compound words are most commonly found in Germanic languages. *Arguments:* -`dictionary`:: (required) The path of a file that contains a list of simple words, one per line. Blank lines and lines that begin with "#" are ignored. See <> for more information. +`dictionary`:: (required) The path of a file that contains a list of simple words, one per line. Blank lines and lines that begin with "#" are ignored. See <> for more information. `minWordSize`:: (integer, default 5) Any token shorter than this is not decompounded. @@ -220,7 +220,7 @@ Unicode Collation in Solr is fast, because all the work is done at index time. Rather than specifying an analyzer within ``, the `solr.CollationField` and `solr.ICUCollationField` field type classes provide this functionality. `solr.ICUCollationField`, which is backed by http://site.icu-project.org[the ICU4J library], provides more flexible configuration, has more locales, is significantly faster, and requires less memory and less index space, since its keys are smaller than those produced by the JDK implementation that backs `solr.CollationField`. -To use `solr.ICUCollationField`, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. +To use `solr.ICUCollationField`, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. `solr.ICUCollationField` and `solr.CollationField` fields can be created in two ways: @@ -487,7 +487,7 @@ The `lucene/analysis/opennlp` module provides OpenNLP integration via several an NOTE: The <> must be used with all other OpenNLP analysis components, for two reasons: first, the OpenNLP Tokenizer detects and marks the sentence boundaries required by all the OpenNLP filters; and second, since the pre-trained OpenNLP models used by these filters were trained using the corresponding language-specific sentence-detection/tokenization models, the same tokenization, using the same models, must be used at runtime for optimal performance. -To use the OpenNLP components, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. +To use the OpenNLP components, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. === OpenNLP Tokenizer @@ -497,9 +497,9 @@ The OpenNLP Tokenizer takes two language-specific binary model files as paramete *Arguments:* -`sentenceModel`:: (required) The path of a language-specific OpenNLP sentence detection model file. See <> for more information. +`sentenceModel`:: (required) The path of a language-specific OpenNLP sentence detection model file. See <> for more information. -`tokenizerModel`:: (required) The path of a language-specific OpenNLP tokenization model file. See <> for more information. +`tokenizerModel`:: (required) The path of a language-specific OpenNLP tokenization model file. See <> for more information. *Example:* @@ -541,7 +541,7 @@ NOTE: Lucene currently does not index token types, so if you want to keep this i *Arguments:* -`posTaggerModel`:: (required) The path of a language-specific OpenNLP POS tagger model file. See <> for more information. +`posTaggerModel`:: (required) The path of a language-specific OpenNLP POS tagger model file. See <> for more information. *Examples:* @@ -636,7 +636,7 @@ NOTE: Lucene currently does not index token types, so if you want to keep this i *Arguments:* -`chunkerModel`:: (required) The path of a language-specific OpenNLP phrase chunker model file. See <> for more information. +`chunkerModel`:: (required) The path of a language-specific OpenNLP phrase chunker model file. See <> for more information. *Examples*: @@ -700,9 +700,9 @@ This filter replaces the text of each token with its lemma. Both a dictionary-ba Either `dictionary` or `lemmatizerModel` must be provided, and both may be provided - see the examples below: -`dictionary`:: (optional) The path of a lemmatization dictionary file. See <> for more information. The dictionary file must be encoded as UTF-8, with one entry per line, in the form `word[tab]lemma[tab]part-of-speech`, e.g., `wrote[tab]write[tab]VBD`. +`dictionary`:: (optional) The path of a lemmatization dictionary file. See <> for more information. The dictionary file must be encoded as UTF-8, with one entry per line, in the form `word[tab]lemma[tab]part-of-speech`, e.g., `wrote[tab]write[tab]VBD`. -`lemmatizerModel`:: (optional) The path of a language-specific OpenNLP lemmatizer model file. See <> for more information. +`lemmatizerModel`:: (optional) The path of a language-specific OpenNLP lemmatizer model file. See <> for more information. *Examples:* @@ -1033,7 +1033,7 @@ Solr can stem Catalan using the Snowball Porter Stemmer with an argument of `lan === Traditional Chinese -The default configuration of the <> is suitable for Traditional Chinese text. It follows the Word Break rules from the Unicode Text Segmentation algorithm for non-Chinese text, and uses a dictionary to segment Chinese words. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See the `solr/contrib/analysis-extras/README.txt` for information on which jars you need to add. +The default configuration of the <> is suitable for Traditional Chinese text. It follows the Word Break rules from the Unicode Text Segmentation algorithm for non-Chinese text, and uses a dictionary to segment Chinese words. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See the `solr/contrib/analysis-extras/README.txt` for information on which jars you need to add. <> can also be used to tokenize Traditional Chinese text. Following the Word Break rules from the Unicode Text Segmentation algorithm, it produces one token per Chinese character. When combined with <>, overlapping bigrams of Chinese characters are formed. @@ -1105,9 +1105,9 @@ See the example under <>. === Simplified Chinese -For Simplified Chinese, Solr provides support for Chinese sentence and word segmentation with the <>. This component includes a large dictionary and segments Chinese text into words with the Hidden Markov Model. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See the `solr/contrib/analysis-extras/README.txt` for information on which jars you need to add. +For Simplified Chinese, Solr provides support for Chinese sentence and word segmentation with the <>. This component includes a large dictionary and segments Chinese text into words with the Hidden Markov Model. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See the `solr/contrib/analysis-extras/README.txt` for information on which jars you need to add. -The default configuration of the <> is also suitable for Simplified Chinese text. It follows the Word Break rules from the Unicode Text Segmentation algorithm for non-Chinese text, and uses a dictionary to segment Chinese words. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See the `solr/contrib/analysis-extras/README.txt` for information on which jars you need to add. +The default configuration of the <> is also suitable for Simplified Chinese text. It follows the Word Break rules from the Unicode Text Segmentation algorithm for non-Chinese text, and uses a dictionary to segment Chinese words. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See the `solr/contrib/analysis-extras/README.txt` for information on which jars you need to add. Also useful for Chinese analysis: @@ -1162,7 +1162,7 @@ Also useful for Chinese analysis: === HMM Chinese Tokenizer -For Simplified Chinese, Solr provides support for Chinese sentence and word segmentation with the `solr.HMMChineseTokenizerFactory` in the `analysis-extras` contrib module. This component includes a large dictionary and segments Chinese text into words with the Hidden Markov Model. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. +For Simplified Chinese, Solr provides support for Chinese sentence and word segmentation with the `solr.HMMChineseTokenizerFactory` in the `analysis-extras` contrib module. This component includes a large dictionary and segments Chinese text into words with the Hidden Markov Model. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. *Factory class:* `solr.HMMChineseTokenizerFactory` @@ -1958,7 +1958,7 @@ Example: [[hebrew-lao-myanmar-khmer]] === Hebrew, Lao, Myanmar, Khmer -Lucene provides support, in addition to UAX#29 word break rules, for Hebrew's use of the double and single quote characters, and for segmenting Lao, Myanmar, and Khmer into syllables with the `solr.ICUTokenizerFactory` in the `analysis-extras` contrib module. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt for` instructions on which jars you need to add. +Lucene provides support, in addition to UAX#29 word break rules, for Hebrew's use of the double and single quote characters, and for segmenting Lao, Myanmar, and Khmer into syllables with the `solr.ICUTokenizerFactory` in the `analysis-extras` contrib module. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt for` instructions on which jars you need to add. See <> for more information. @@ -2165,7 +2165,7 @@ Solr includes support for normalizing Persian, and Lucene includes an example st === Polish -Solr provides support for Polish stemming with the `solr.StempelPolishStemFilterFactory`, and `solr.MorphologikFilterFactory` for lemmatization, in the `contrib/analysis-extras` module. The `solr.StempelPolishStemFilterFactory` component includes an algorithmic stemmer with tables for Polish. To use either of these filters, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. +Solr provides support for Polish stemming with the `solr.StempelPolishStemFilterFactory`, and `solr.MorphologikFilterFactory` for lemmatization, in the `contrib/analysis-extras` module. The `solr.StempelPolishStemFilterFactory` component includes an algorithmic stemmer with tables for Polish. To use either of these filters, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. *Factory class:* `solr.StempelPolishStemFilterFactory` and `solr.MorfologikFilterFactory` @@ -2682,7 +2682,7 @@ Solr includes support for stemming Turkish with the `solr.SnowballPorterFilterFa === Ukrainian -Solr provides support for Ukrainian lemmatization with the `solr.MorphologikFilterFactory`, in the `contrib/analysis-extras` module. To use this filter, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. +Solr provides support for Ukrainian lemmatization with the `solr.MorphologikFilterFactory`, in the `contrib/analysis-extras` module. To use this filter, you must add additional .jars to Solr's classpath (as described in the section <>). See `solr/contrib/analysis-extras/README.txt` for instructions on which jars you need to add. Lucene also includes an example Ukrainian stopword list, in the `lucene-analyzers-morfologik` jar. diff --git a/solr/solr-ref-guide/src/learning-to-rank.adoc b/solr/solr-ref-guide/src/learning-to-rank.adoc index 4f550cded7f..a0488a6c5db 100644 --- a/solr/solr-ref-guide/src/learning-to-rank.adoc +++ b/solr/solr-ref-guide/src/learning-to-rank.adoc @@ -533,7 +533,7 @@ Assuming that you consider to use a large model placed at `/path/to/models/myMod } ---- -First, add the directory to Solr's resource paths with a <` directive>>: +First, add the directory to Solr's resource paths with a <` directive>>: [source,xml] ---- diff --git a/solr/solr-ref-guide/src/libs.adoc b/solr/solr-ref-guide/src/libs.adoc new file mode 100644 index 00000000000..91f9bb81db9 --- /dev/null +++ b/solr/solr-ref-guide/src/libs.adoc @@ -0,0 +1,78 @@ += Lib Directories and Directives + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Here we describe two simple and effective methods to make the `.jar` files for Solr plugins visible to Solr. + +Such files are sometimes called "libraries" or "libs" for short. +Essentially you can put them in some special places or explicitly tell Solr about them from your config. + +If there is overlap or inter-dependencies between libraries, then pay attention to the order. You can think of it like a stack that is searched top-down. At the top are the lib directives in reverse order, then Solr core's lib, then Solr home's lib, then Solr itself. + +== Lib Directories + +There are several special places you can place Solr plugin `.jar` files: + +* `/lib/`: The `.jar` files placed here are available to all Solr cores running on the node, and to node level plugins referenced in `solr.xml` -- so basically everything. +This directory is not present by default so create it. +See <>. + +* `/lib/`: In standalone Solr, you may want to add plugins just for a specific Solr core. +Create this adjacent to the `conf/` directory; it's not present by default. + +* `/server/solr-webapp/webapp/WEB-INF/lib/`: The `.jar` files for Solr itself and it's dependencies live here. +Certain plugins or add-ons to plugins require placement here. +They will document themselves to say so. + +Solr incorporates Jetty for providing HTTP server functionality. +Jetty has some directories that contain `.jar` files for itself and its own plugins / modules or JVM level plugins (e.g. loggers). +Solr plugins won't work in these locations. + +== Lib Directives in SolrConfig + +_Both_ plugin and <> file paths are configurable via `` directives in `solrconfig.xml`. +When a directive matches a directory, then resources can be resolved from it. +When a directive matches a `.jar` file, Solr plugins and their dependencies are resolved from it. +Resources can be placed in a `.jar` too but that's unusual. +It's erroneous to refer to any other type of file. + +A `` directive must have one (not both) of these two attributes: + +* `path`: used to refer to a single directory (for resources) or file (for a plugin `.jar`) + +* `dir`: used to refer to _all_ direct descendants of the specified directory. Optionally supply a `regex` attribute to filter these to those matching the regular expression. + +All directories are resolved as relative to the Solr core's `instanceDir`. + +These examples show how to load contrib modules into Solr: + +[source,xml] +---- + + + + + + + + + + + + +---- diff --git a/solr/solr-ref-guide/src/resource-and-plugin-loading.adoc b/solr/solr-ref-guide/src/resource-and-plugin-loading.adoc deleted file mode 100644 index 6efd1353290..00000000000 --- a/solr/solr-ref-guide/src/resource-and-plugin-loading.adoc +++ /dev/null @@ -1,86 +0,0 @@ -= Resource and Plugin Loading -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -Solr components can be configured using *resources*: data stored in external files that may be referred to in a location-independent fashion. Examples include: files needed by schema components, e.g., a stopword list for <>; and machine-learned models for <>. - -Solr *plugins*, which can be configured in `solrconfig.xml`, are Java classes that are normally packaged in `.jar` files along with supporting classes and data. Solr ships with a number of built-in plugins, and can also be configured to use custom plugins. Example plugins are the <> and custom search components. - -Resources and plugins may be stored: - -* in ZooKeeper under a collection's configset node (SolrCloud only); -* on a filesystem accessible to Solr nodes; or -* in Solr's <> (SolrCloud only). - -NOTE: Schema components may not be stored as plugins in the Blob Store, and cannot access resources stored in the Blob Store. - -== Resource and Plugin Loading Sequence - -Under SolrCloud, resources and plugins to be loaded are first looked up in ZooKeeper under the collection's configset znode. If the resource or plugin is not found there, Solr will fall back to loading <>. - -Note that by default, Solr will not attempt to load resources and plugins from the Blob Store. To enable this, see the section <>. When loading from the Blob Store is enabled for a component, lookups occur only in the Blob Store, and never in ZooKeeper or on the filesystem. - -== Resources and Plugins in ConfigSets on ZooKeeper - -Resources and plugins may be uploaded to ZooKeeper as part of a configset, either via the <> or <>. - -To upload a plugin or resource to a configset already stored on ZooKeeper, you can use <>. - -CAUTION: By default, ZooKeeper's file size limit is 1MB. If your files are larger than this, you'll need to either <> or store them instead <>. - -== Resources and Plugins on the Filesystem - -Under standalone Solr, when looking up a plugin or resource to be loaded, Solr's resource loader will first look under the `/conf/` directory. If the plugin or resource is not found, the configured plugin and resource file paths are searched - see the section <> below. - -On core load, Solr's resource loader constructs a list of paths (subdirectories and jars), first under <>, and then under directories pointed to by <` directives in SolrConfig>>. - -When looking up a resource or plugin to be loaded, the paths on the list are searched in the order they were added. - -NOTE: Under SolrCloud, each node hosting a collection replica will need its own copy of plugins and resources to be loaded. - -To get Solr's resource loader to find resources either under subdirectories or in jar files that were created after Solr's resource path list was constructed, reload the collection (SolrCloud) or the core (standalone Solr). Restarting all affected Solr nodes also works. - -WARNING: Resource files *will not be loaded* if they are located directly under either `solr_home/lib` or a directory given by the `dir` attribute on a `` directive in SolrConfig. Resources are only searched for under subdirectories or in jar files found in those locations. - -=== solr_home/lib - -Each Solr node can have a directory named `lib/` under the <>. In order to use this directory to host resources or plugins, it must first be manually created. - -=== Lib Directives in SolrConfig - -Plugin and resource file paths are configurable via `` directives in `solrconfig.xml`. - -Loading occurs in the order `` directives appear in `solrconfig.xml`. If there are dependencies, list the lowest level dependency jar first. - -A regular expression supplied in the `` element's `regex` attribute value can be used to restrict which subdirectories and/or jar files are added to the Solr resource loader's list of search locations. If no regular expression is given, all direct subdirectory and jar children are included in the resource path list. All directories are resolved as relative to the Solr core's `instanceDir`. - -From an example SolrConfig: - -[source,xml] ----- - - - - - - - - - - - ----- diff --git a/solr/solr-ref-guide/src/resource-loading.adoc b/solr/solr-ref-guide/src/resource-loading.adoc new file mode 100644 index 00000000000..944fb37a954 --- /dev/null +++ b/solr/solr-ref-guide/src/resource-loading.adoc @@ -0,0 +1,44 @@ += Resource Loading + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Solr components can be configured using *resources*: data stored in external files that may be referred to in a location-independent fashion. + +Examples of resources include: files needed by schema components, e.g., a stopword list for <>; and machine-learned models for <>. +_Resources are typically resolved from the configSet_ but there are other options too. + +Solr's resources are generally only loaded initially when the Solr collection or Solr core is loaded. +After you update a resource, you'll typically need to _reload_ the affected collections (SolrCloud) or the cores (standalone Solr). +Restarting all affected Solr nodes also works. +<> can be manipulated via APIs and do not need an explicit reload. + +== Resources in ConfigSets + +<> are the directories containing solrconfig.xml, the schema, and resources referenced by them. +In SolrCloud they are in ZooKeeper whereas in standalone they are on the file system. +In either mode, configSets might be shared or might be dedicated to a configSet. +Prefer to put resources here. + +== Resources in Other Places + +Resources can also be placed in an arbitrary directory and <> from a `` directive in `solrconfig.xml`, provided the directive refers to a directory and not the actual resource file. Example: `` +This choice may make sense if the resource is too large for a configSet in ZooKeeper. +However it's up to you to somehow ensure that all nodes in your cluster have access to these resources. + +Finally, and this is very unusual, resources can also be packaged inside `.jar` files from which they will be referenced. +That might make sense for default resources wherein a plugin user can override it via placing the same-named file in a configSet. \ No newline at end of file diff --git a/solr/solr-ref-guide/src/solr-plugins.adoc b/solr/solr-ref-guide/src/solr-plugins.adoc index 5e96b3ceb1d..b3693cfe75b 100644 --- a/solr/solr-ref-guide/src/solr-plugins.adoc +++ b/solr/solr-ref-guide/src/solr-plugins.adoc @@ -1,5 +1,8 @@ = Solr Plugins -:page-children: adding-custom-plugins-in-solrcloud-mode +:page-children: libs, \ + package-manager, \ + adding-custom-plugins-in-solrcloud-mode + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -17,8 +20,37 @@ // specific language governing permissions and limitations // under the License. -Solr allows you to load custom code to perform a variety of tasks within Solr, from custom Request Handlers to process your searches, to custom Analyzers and Token Filters for your text field. You can even load custom Field Types. These pieces of custom code are called _plugins_. +One of Solr's strengths is providing a rich platform of functionality with the option of adding your own custom components running within Solr. -Not everyone will need to create plugins for their Solr instances - what's provided is usually enough for most applications. However, if there's something that you need, you may want to review the Solr Wiki documentation on plugins at https://cwiki.apache.org/confluence/display/solr/SolrPlugins[SolrPlugins]. +Solr calls such components *plugins* when the implementation is configurable. +Surely you have seen many already throughout Solr's configuration via the "class" reference. +Common examples are Request Handlers, Search Components, and Query Parsers to process your searches, and Token Filters for processing text. -If you have a plugin you would like to use, and you are running in SolrCloud mode, you can use the Blob Store API and the Config API to load the jars to Solr. The commands to use are described in the section <>. +Most apps don't need to create plugins because Solr offers a rich set of them built-in. +However if you do, start by looking at the code for existing similar plugins. +Writing your own is an advanced subject that is out of scope of the reference guide. +One resource is the Solr Wiki documentation on plugins at https://cwiki.apache.org/confluence/display/solr/SolrPlugins[SolrPlugins], which is rather out-of-date but has some utility. + +== Installing Plugins == + +Most plugins are built-in to Solr and there is nothing to install. +The subject here is how to make other plugins available to Solr, including those in contrib modules. +Plugins are packaged into a Java jar file and may have other dependent jar files to function. + +The next sections describe some options: + +* <>: +Describes where to put the plugin's JAR files on the file system; either in one of the special places or a place convenient to you along with a `` directive in `solrconfig.xml`. +This has been the standard approach since Solr's inception. +It's simple and reliable but it's entirely on you to ensure that all nodes in a cluster have them. +Contrib modules ship with Solr so there's no effort for them but not so for other plugins (yours or 3rd party). + +* <>: +Describes a new and experimental system to manage packages of plugins in SolrCloud. +It includes CLI commands, cluster-wide installation, use of plugin registries that host plugins, cryptographically signed plugins for security, and more. +Only some plugins support this. + +* <>: +Describes a deprecated system that predates the above package management system. +It's functionality is a subset of the package management system. +It will no longer be supported in Solr 9. diff --git a/solr/solr-ref-guide/src/the-well-configured-solr-instance.adoc b/solr/solr-ref-guide/src/the-well-configured-solr-instance.adoc index 85531e0f3b8..546f2c6944e 100644 --- a/solr/solr-ref-guide/src/the-well-configured-solr-instance.adoc +++ b/solr/solr-ref-guide/src/the-well-configured-solr-instance.adoc @@ -1,5 +1,12 @@ = The Well-Configured Solr Instance -:page-children: configuring-solrconfig-xml, solr-cores-and-solr-xml, configuration-apis, implicit-requesthandlers, solr-plugins, jvm-settings, v2-api, package-manager +:page-children: configuring-solrconfig-xml, \ + solr-cores-and-solr-xml, \ + resource-loading, \ + configuration-apis, \ + implicit-requesthandlers, \ + jvm-settings, \ + v2-api + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -25,14 +32,12 @@ This section covers the following topics: <>: Describes how to work with `solr.xml` and `core.properties` to configure your Solr core, or multiple Solr cores within a single instance. +<>: Describes how word lists, model files, and other related data are resolved by the components that need them. + <>: Describes several APIs used to configure Solr: Blob Store, Config, Request Parameters and Managed Resources. <>: Describes various end-points automatically provided by Solr and how to configure them. -<>: Introduces Solr plugins with pointers to more information. - -<>: Installing, deploying and updating packages (containing plugins) into a Solr cluster - <>: Gives some guidance on best practices for working with Java Virtual Machines. <>: Describes how to use the new V2 APIs, a redesigned API framework covering most Solr APIs. diff --git a/solr/solr-ref-guide/src/tokenizers.adoc b/solr/solr-ref-guide/src/tokenizers.adoc index c883342debe..843c0fe8623 100644 --- a/solr/solr-ref-guide/src/tokenizers.adoc +++ b/solr/solr-ref-guide/src/tokenizers.adoc @@ -516,7 +516,7 @@ The default configuration for `solr.ICUTokenizerFactory` provides UAX#29 word br [IMPORTANT] ==== -To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See the `solr/contrib/analysis-extras/README.txt` for information on which jars you need to add. +To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <>). See the `solr/contrib/analysis-extras/README.txt` for information on which jars you need to add. ==== diff --git a/solr/solr-ref-guide/src/update-request-processors.adoc b/solr/solr-ref-guide/src/update-request-processors.adoc index 27b999bc7eb..e459ddf4a82 100644 --- a/solr/solr-ref-guide/src/update-request-processors.adoc +++ b/solr/solr-ref-guide/src/update-request-processors.adoc @@ -353,7 +353,7 @@ The {solr-javadocs}/solr-langid/index.html[`langid`] contrib provides:: The {solr-javadocs}/solr-analysis-extras/index.html[`analysis-extras`] contrib provides:: -{solr-javadocs}/solr-analysis-extras/org/apache/solr/update/processor/OpenNLPExtractNamedEntitiesUpdateProcessorFactory.html[OpenNLPExtractNamedEntitiesUpdateProcessorFactory]::: Update document(s) to be indexed with named entities extracted using an OpenNLP NER model. Note that in order to use model files larger than 1MB on SolrCloud, you must either <> or <> on each node hosting a collection replica. +{solr-javadocs}/solr-analysis-extras/org/apache/solr/update/processor/OpenNLPExtractNamedEntitiesUpdateProcessorFactory.html[OpenNLPExtractNamedEntitiesUpdateProcessorFactory]::: Update document(s) to be indexed with named entities extracted using an OpenNLP NER model. Note that in order to use model files larger than 1MB on SolrCloud, you must either <> or <> on each node hosting a collection replica. === Update Processor Factories You Should _Not_ Modify or Remove From 640ff96522eeff891d10cbc49bc8eadfcb0826f1 Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Sun, 15 Dec 2019 10:44:02 +0530 Subject: [PATCH 10/15] SOLR-13662: Fixes to package manager * Better logging and error reporting * Fixing deploy command to handle previously undeployed packages * Test now uses @LogLevel annotation * Deploy command had a hard coded collection name by mistake, fix it --- .../DefaultPackageRepository.java | 5 +- .../solr/packagemanager/PackageManager.java | 85 +++++++++++++------ .../solr/packagemanager/PackageUtils.java | 7 ++ .../packagemanager/RepositoryManager.java | 30 ++++--- .../org/apache/solr/util/PackageTool.java | 18 ++-- .../solr/cloud/PackageManagerCLITest.java | 9 +- 6 files changed, 100 insertions(+), 54 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java b/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java index dee55a5047c..b55dc717337 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.solr.common.SolrException; @@ -82,8 +83,8 @@ public class DefaultPackageRepository extends PackageRepository { public Path download(String artifactName) throws SolrException, IOException { Path tmpDirectory = Files.createTempDirectory("solr-packages"); tmpDirectory.toFile().deleteOnExit(); - URL url = new URL(new URL(repositoryURL), artifactName); - String fileName = url.getPath().substring(url.getPath().lastIndexOf('/') + 1); + URL url = new URL(new URL(repositoryURL.endsWith("/")? repositoryURL: repositoryURL+"/"), artifactName); + String fileName = FilenameUtils.getName(url.getPath()); Path destination = tmpDirectory.resolve(fileName); switch (url.getProtocol()) { diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java index 73f50dc0cfa..0d7a7d61829 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -26,15 +26,19 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Scanner; +import java.util.Set; +import java.util.stream.Collectors; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.common.NavigableObject; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.Utils; import org.apache.solr.packagemanager.SolrPackage.Command; import org.apache.solr.packagemanager.SolrPackage.Manifest; @@ -84,9 +88,9 @@ public class PackageManager implements Closeable { try { Map packagesZnodeMap = null; - if (zkClient.exists("/packages.json", true) == true) { + if (zkClient.exists(ZkStateReader.SOLR_PKGS_PATH, true) == true) { packagesZnodeMap = (Map)getMapper().readValue( - new String(zkClient.getData("/packages.json", null, null, true), "UTF-8"), Map.class).get("packages"); + new String(zkClient.getData(ZkStateReader.SOLR_PKGS_PATH, null, null, true), "UTF-8"), Map.class).get("packages"); for (Object packageName: packagesZnodeMap.keySet()) { List pkg = (List)packagesZnodeMap.get(packageName); for (Map pkgVersion: (List)pkg) { @@ -112,34 +116,51 @@ public class PackageManager implements Closeable { Map packages = null; try { NavigableObject result = (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), - solrBaseUrl+"/api/collections/"+collection+"/config/params/PKG_VERSIONS?omitHeader=true&wt=javabin", Utils.JAVABINCONSUMER); + solrBaseUrl + PackageUtils.getCollectionParamsPath(collection) + "/PKG_VERSIONS?omitHeader=true&wt=javabin", Utils.JAVABINCONSUMER); packages = (Map) result._get("/response/params/PKG_VERSIONS", Collections.emptyMap()); } catch (PathNotFoundException ex) { // Don't worry if PKG_VERSION wasn't found. It just means this collection was never touched by the package manager. } if (packages == null) return Collections.emptyMap(); - Map ret = new HashMap(); + Map ret = new HashMap<>(); for (String packageName: packages.keySet()) { - if (Strings.isNullOrEmpty(packageName) == false) { // There can be an empty key, storing the version here + if (Strings.isNullOrEmpty(packageName) == false && // There can be an empty key, storing the version here + packages.get(packageName) != null) { // null means the package was undeployed from this package before ret.put(packageName, getPackageInstance(packageName, packages.get(packageName))); } } return ret; } + private void ensureCollectionsExist(List collections) { + try { + List existingCollections = zkClient.getChildren("/collections", null, true); + Set nonExistent = new HashSet<>(collections); + nonExistent.removeAll(existingCollections); + if (nonExistent.isEmpty() == false) { + throw new SolrException(ErrorCode.BAD_REQUEST, "Collection(s) doesn't exist: " + nonExistent.toString()); + } + } catch (KeeperException | InterruptedException e) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to fetch list of collections from ZK."); + } + } + private boolean deployPackage(SolrPackageInstance packageInstance, boolean pegToLatest, boolean isUpdate, boolean noprompt, List collections, String overrides[]) { - for (String collection: collections) { + List previouslyDeployed = new ArrayList<>(); // collections where package is already deployed in + for (String collection: collections) { SolrPackageInstance deployedPackage = getPackagesDeployed(collection).get(packageInstance.name); if (packageInstance.equals(deployedPackage)) { if (!pegToLatest) { PackageUtils.printRed("Package " + packageInstance + " already deployed on "+collection); + previouslyDeployed.add(collection); continue; } } else { if (deployedPackage != null && !isUpdate) { PackageUtils.printRed("Package " + deployedPackage + " already deployed on "+collection+". To update to "+packageInstance+", pass --update parameter."); + previouslyDeployed.add(collection); continue; } } @@ -148,9 +169,9 @@ public class PackageManager implements Closeable { // Get package params try { - boolean packageParamsExist = ((Map)PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + "/api/collections/abc/config/params/packages", Map.class) + boolean packageParamsExist = ((Map)PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + PackageUtils.getCollectionParamsPath(collection) + "/packages", Map.class) .getOrDefault("response", Collections.emptyMap())).containsKey("params"); - SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", + SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection), getMapper().writeValueAsString(Collections.singletonMap(packageParamsExist? "update": "set", Collections.singletonMap("packages", Collections.singletonMap(packageInstance.name, collectionParameterOverrides))))); } catch (Exception e) { @@ -159,7 +180,7 @@ public class PackageManager implements Closeable { // Set the package version in the collection's parameters try { - SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", + SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection), "{set:{PKG_VERSIONS:{" + packageInstance.name+": '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version)+"'}}}"); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); @@ -168,7 +189,7 @@ public class PackageManager implements Closeable { // If updating, refresh the package version for this to take effect if (isUpdate || pegToLatest) { try { - SolrCLI.postJsonToSolr(solrClient, "/api/cluster/package", "{\"refresh\": \"" + packageInstance.name + "\"}"); + SolrCLI.postJsonToSolr(solrClient, PackageUtils.PACKAGE_PATH, "{\"refresh\": \"" + packageInstance.name + "\"}"); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } @@ -213,19 +234,27 @@ public class PackageManager implements Closeable { // Set the package version in the collection's parameters try { - SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", + SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection), "{update:{PKG_VERSIONS:{'" + packageInstance.name + "' : '" + (pegToLatest? PackagePluginHolder.LATEST: packageInstance.version) + "'}}}"); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } } - // Verify that package was successfully deployed - boolean success = verify(packageInstance, collections); - if (success) { - PackageUtils.printGreen("Deployed and verified package: " + packageInstance.name + ", version: " + packageInstance.version); + List deployedCollections = collections.stream().filter(c -> !previouslyDeployed.contains(c)).collect(Collectors.toList()); + + boolean success = true; + if (deployedCollections.isEmpty() == false) { + // Verify that package was successfully deployed + success = verify(packageInstance, deployedCollections); + if (success) { + PackageUtils.printGreen("Deployed on " + deployedCollections + " and verified package: " + packageInstance.name + ", version: " + packageInstance.version); + } } - return success; + if (previouslyDeployed.isEmpty() == false) { + PackageUtils.printRed("Already Deployed on " + previouslyDeployed + ", package: " + packageInstance.name + ", version: " + packageInstance.version); + } + return previouslyDeployed.isEmpty() && success; } private Map getCollectionParameterOverrides(SolrPackageInstance packageInstance, boolean isUpdate, @@ -243,7 +272,7 @@ public class PackageManager implements Closeable { Map getPackageParams(String packageName, String collection) { try { return (Map)((Map)((Map)((Map) - PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + "/api/collections/" + collection + "/config/params/packages", Map.class) + PackageUtils.getJson(solrClient.getHttpClient(), solrBaseUrl + PackageUtils.getCollectionParamsPath(collection) + "/packages", Map.class) .get("response")) .get("params")) .get("packages")).get(packageName); @@ -260,7 +289,6 @@ public class PackageManager implements Closeable { public boolean verify(SolrPackageInstance pkg, List collections) { boolean success = true; for (Plugin plugin: pkg.plugins) { - PackageUtils.printGreen(plugin.verifyCommand); for (String collection: collections) { Map collectionParameterOverrides = getPackageParams(pkg.name, collection); Command cmd = plugin.verifyCommand; @@ -322,6 +350,8 @@ public class PackageManager implements Closeable { */ public void deploy(String packageName, String version, String[] collections, String[] parameters, boolean isUpdate, boolean noprompt) throws SolrException { + ensureCollectionsExist(Arrays.asList(collections)); + boolean pegToLatest = PackageUtils.LATEST.equals(version); // User wants to peg this package's version to the latest installed (for auto-update, i.e. no explicit deploy step) SolrPackageInstance packageInstance = getPackageInstance(packageName, version); if (packageInstance == null) { @@ -342,11 +372,17 @@ public class PackageManager implements Closeable { } /** - * Undeploys a packge from given collections. + * Undeploys a package from given collections. */ public void undeploy(String packageName, String[] collections) throws SolrException { + ensureCollectionsExist(Arrays.asList(collections)); + for (String collection: collections) { SolrPackageInstance deployedPackage = getPackagesDeployed(collection).get(packageName); + if (deployedPackage == null) { + PackageUtils.printRed("Package "+packageName+" not deployed on collection "+collection); + continue; + } Map collectionParameterOverrides = getPackageParams(packageName, collection); // Run the uninstall command for all plugins @@ -374,13 +410,14 @@ public class PackageManager implements Closeable { // Set the package version in the collection's parameters try { - SolrCLI.postJsonToSolr(solrClient, "/api/collections/" + collection + "/config/params", "{set: {PKG_VERSIONS: {"+packageName+": null}}}"); - SolrCLI.postJsonToSolr(solrClient, "/api/cluster/package", "{\"refresh\": \"" + packageName + "\"}"); + SolrCLI.postJsonToSolr(solrClient, PackageUtils.getCollectionParamsPath(collection), + "{set: {PKG_VERSIONS: {"+packageName+": null}}}"); // Is it better to "unset"? If so, build support in params API for "unset" + SolrCLI.postJsonToSolr(solrClient, PackageUtils.PACKAGE_PATH, "{\"refresh\": \"" + packageName + "\"}"); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } - // TODO: Also better to remove the package parameters + // TODO: Also better to remove the package parameters PKG_VERSION etc. } } @@ -391,14 +428,14 @@ public class PackageManager implements Closeable { public Map getDeployedCollections(String packageName) { List allCollections; try { - allCollections = zkClient.getChildren("/collections", null, true); + allCollections = zkClient.getChildren(ZkStateReader.COLLECTIONS_ZKNODE, null, true); } catch (KeeperException | InterruptedException e) { throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, e); } Map deployed = new HashMap(); for (String collection: allCollections) { // Check package version installed - String paramsJson = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrBaseUrl + "/api/collections/" + collection + "/config/params/PKG_VERSIONS?omitHeader=true"); + String paramsJson = PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrBaseUrl + PackageUtils.getCollectionParamsPath(collection) + "/PKG_VERSIONS?omitHeader=true"); String version = null; try { version = JsonPath.parse(paramsJson, PackageUtils.jsonPathConfiguration()) diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java index 77ae057ca86..11859a277eb 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java @@ -58,6 +58,9 @@ public class PackageUtils { * Represents a version which denotes the latest version available at the moment. */ public static String LATEST = "latest"; + + public static String PACKAGE_PATH = "/api/cluster/package"; + public static String REPOSITORIES_ZK_PATH = "/repositories.json"; public static Configuration jsonPathConfiguration() { MappingProvider provider = new JacksonMappingProvider(); @@ -235,4 +238,8 @@ public class PackageUtils { } return collections; } + + public static String getCollectionParamsPath(String collection) { + return "/api/collections/" + collection + "/config/params"; + } } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 5dd503ebe0c..3d6075b42a7 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -122,23 +122,23 @@ public class RepositoryManager { List repos = getMapper().readValue(existingRepositoriesJson, List.class); repos.add(new DefaultPackageRepository(name, uri)); - if (packageManager.zkClient.exists("/repositories.json", true) == false) { - packageManager.zkClient.create("/repositories.json", getMapper().writeValueAsString(repos).getBytes("UTF-8"), CreateMode.PERSISTENT, true); + if (packageManager.zkClient.exists(PackageUtils.REPOSITORIES_ZK_PATH, true) == false) { + packageManager.zkClient.create(PackageUtils.REPOSITORIES_ZK_PATH, getMapper().writeValueAsString(repos).getBytes("UTF-8"), CreateMode.PERSISTENT, true); } else { - packageManager.zkClient.setData("/repositories.json", getMapper().writeValueAsString(repos).getBytes("UTF-8"), true); + packageManager.zkClient.setData(PackageUtils.REPOSITORIES_ZK_PATH, getMapper().writeValueAsString(repos).getBytes("UTF-8"), true); } if (packageManager.zkClient.exists("/keys", true)==false) packageManager.zkClient.create("/keys", new byte[0], CreateMode.PERSISTENT, true); if (packageManager.zkClient.exists("/keys/exe", true)==false) packageManager.zkClient.create("/keys/exe", new byte[0], CreateMode.PERSISTENT, true); - if (packageManager.zkClient.exists("/keys/exe/"+name+".der", true)==false) { - packageManager.zkClient.create("/keys/exe/"+name+".der", new byte[0], CreateMode.PERSISTENT, true); + if (packageManager.zkClient.exists("/keys/exe/" + name + ".der", true)==false) { + packageManager.zkClient.create("/keys/exe/" + name + ".der", new byte[0], CreateMode.PERSISTENT, true); } - packageManager.zkClient.setData("/keys/exe/"+name+".der", IOUtils.toByteArray(new URL(uri+"/publickey.der").openStream()), true); + packageManager.zkClient.setData("/keys/exe/" + name + ".der", IOUtils.toByteArray(new URL(uri + "/publickey.der").openStream()), true); } private String getRepositoriesJson(SolrZkClient zkClient) throws UnsupportedEncodingException, KeeperException, InterruptedException { - if (zkClient.exists("/repositories.json", true)) { - return new String(zkClient.getData("/repositories.json", null, null, true), "UTF-8"); + if (zkClient.exists(PackageUtils.REPOSITORIES_ZK_PATH, true)) { + return new String(zkClient.getData(PackageUtils.REPOSITORIES_ZK_PATH, null, null, true), "UTF-8"); } return "[]"; } @@ -195,7 +195,7 @@ public class RepositoryManager { add.manifest = "/package/" + packageName + "/" + version + "/manifest.json"; add.manifestSHA512 = manifestSHA512; - V2Request req = new V2Request.Builder("/api/cluster/package") + V2Request req = new V2Request.Builder(PackageUtils.PACKAGE_PATH) .forceV2(true) .withMethod(SolrRequest.METHOD.POST) .withPayload(Collections.singletonMap("add", add)) @@ -308,10 +308,12 @@ public class RepositoryManager { installPackage(packageName, version); } - SolrPackageInstance updatedPackage = packageManager.getPackageInstance(packageName, PackageUtils.LATEST); - boolean res = packageManager.verify(updatedPackage, peggedToLatest); - PackageUtils.printGreen("Verifying version " + updatedPackage.version + - " on " + peggedToLatest + ", result: " + res); - if (!res) throw new SolrException(ErrorCode.BAD_REQUEST, "Failed verification after deployment"); + if (peggedToLatest.isEmpty() == false) { + SolrPackageInstance updatedPackage = packageManager.getPackageInstance(packageName, PackageUtils.LATEST); + boolean res = packageManager.verify(updatedPackage, peggedToLatest); + PackageUtils.printGreen("Verifying version " + updatedPackage.version + + " on " + peggedToLatest + ", result: " + res); + if (!res) throw new SolrException(ErrorCode.BAD_REQUEST, "Failed verification after deployment"); + } } } diff --git a/solr/core/src/java/org/apache/solr/util/PackageTool.java b/solr/core/src/java/org/apache/solr/util/PackageTool.java index 8464ec14597..f7744c98faf 100644 --- a/solr/core/src/java/org/apache/solr/util/PackageTool.java +++ b/solr/core/src/java/org/apache/solr/util/PackageTool.java @@ -118,9 +118,13 @@ public class PackageTool extends SolrCLI.ToolBase { } else { String packageName = cli.getArgs()[1]; Map deployedCollections = packageManager.getDeployedCollections(packageName); - PackageUtils.printGreen("Collections on which package " + packageName + " was deployed:"); - for (String collection: deployedCollections.keySet()) { - PackageUtils.printGreen("\t" + collection + "("+packageName+":"+deployedCollections.get(collection)+")"); + if (deployedCollections.isEmpty() == false) { + PackageUtils.printGreen("Collections on which package " + packageName + " was deployed:"); + for (String collection: deployedCollections.keySet()) { + PackageUtils.printGreen("\t" + collection + "("+packageName+":"+deployedCollections.get(collection)+")"); + } + } else { + PackageUtils.printGreen("Package "+packageName+" not deployed on any collection."); } } break; @@ -130,7 +134,7 @@ public class PackageTool extends SolrCLI.ToolBase { String packageName = parsedVersion.first(); String version = parsedVersion.second(); repositoryManager.install(packageName, version); - PackageUtils.printGreen(repositoryManager.toString() + " installed."); + PackageUtils.printGreen(packageName + " installed."); break; } case "deploy": @@ -140,7 +144,7 @@ public class PackageTool extends SolrCLI.ToolBase { String version = parsedVersion.second(); boolean noprompt = cli.hasOption('y'); boolean isUpdate = cli.hasOption("update") || cli.hasOption('u'); - packageManager.deploy(packageName, version, PackageUtils.validateCollections(cli.getOptionValues("collections")), cli.getOptionValues("param"), isUpdate, noprompt); + packageManager.deploy(packageName, version, PackageUtils.validateCollections(cli.getOptionValue("collections").split(",")), cli.getOptionValues("param"), isUpdate, noprompt); break; } case "undeploy": @@ -150,7 +154,7 @@ public class PackageTool extends SolrCLI.ToolBase { throw new SolrException(ErrorCode.BAD_REQUEST, "Only package name expected, without a version. Actual: " + cli.getArgList().get(1)); } String packageName = parsedVersion.first(); - packageManager.undeploy(packageName, cli.getOptionValues("collections")); + packageManager.undeploy(packageName, cli.getOptionValue("collections").split(",")); break; } case "help": @@ -228,7 +232,7 @@ public class PackageTool extends SolrCLI.ToolBase { OptionBuilder .withArgName("COLLECTIONS") - .hasArgs() + .hasArg() .isRequired(false) .withDescription("List of collections. Run './solr package help' for more details.") .create("collections"), diff --git a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java index 4c08aad2eca..38b66b60f39 100644 --- a/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java +++ b/solr/core/src/test/org/apache/solr/cloud/PackageManagerCLITest.java @@ -20,11 +20,9 @@ package org.apache.solr.cloud; import java.lang.invoke.MethodHandles; import java.util.Arrays; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.core.config.Configurator; -import org.apache.lucene.util.SuppressForbidden; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.core.TestSolrConfigHandler; +import org.apache.solr.util.LogLevel; import org.apache.solr.util.PackageTool; import org.apache.solr.util.SolrCLI; import org.eclipse.jetty.server.Handler; @@ -39,6 +37,7 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@LogLevel("org.apache=INFO") public class PackageManagerCLITest extends SolrCloudTestCase { // Note for those who want to modify the jar files used in the packages used in this test: @@ -70,13 +69,9 @@ public class PackageManagerCLITest extends SolrCloudTestCase { } @Test - @SuppressForbidden(reason = "Need to turn off logging, and SLF4J doesn't seem to provide for a way.") public void testPackageManager() throws Exception { PackageTool tool = new PackageTool(); - // Enable the logger for this test. Need to do this since the tool disables logger. - Configurator.setRootLevel(Level.INFO); - String solrUrl = cluster.getJettySolrRunner(0).getBaseUrl().toString(); run(tool, new String[] {"-solrUrl", solrUrl, "list-installed"}); From be0b963a2280c3b79fab11ca9370a06e7c7beb39 Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Sun, 15 Dec 2019 11:21:10 +0530 Subject: [PATCH 11/15] SOLR-14071: Updating upgrade notice --- solr/CHANGES.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 05ddf486139..14a17a688d2 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -177,6 +177,9 @@ Upgrade Notes * SOLR-14071: Untrusted configsets (ones that are uploaded via unsecured configset API) cannot use directive. Consider enabling authentication/authorization so that the uploaded configsets are trusted. + Note: If you already have a collection using untrusted configset that uses directive, it will not load after + upgrading to 8.4. You can re-upload your configset using "bin/solr zk -upconfig .." or place your libraries in the + classpath and restart Solr. * SOLR-14065: VelocityResponseWriter has been deprecated and may be removed in a future version. @@ -311,7 +314,7 @@ Bug Fixes * SOLR-13945: Fix: SPLITSHARD can cause data loss on a failure to commit after the sub-shards are active and a rollback is done to make parent shard active again (Ishan Chattopadhyaya, ab) -* SOLR-14071: Untrusted configsets cannot use directive (Ishan Chattopadhyaya) +* SOLR-14071: Untrusted configsets cannot use directive due to security reasons (Ishan Chattopadhyaya) * SOLR-14013: FIX: javabin performance regressions (noble, yonik, Houston Putman) From 2f051a4bfea12ea2c66d16680968697ca4573884 Mon Sep 17 00:00:00 2001 From: Kevin Risden Date: Sat, 14 Dec 2019 14:55:14 -0500 Subject: [PATCH 12/15] SOLR-14086: Tika ClassNotFound error due to commons-compress in solr-core dependency Introduced in SOLR-14033 by including commons-compress as a compile time dependency in Solr core instead of as as test only dependency. Signed-off-by: Kevin Risden --- solr/core/ivy.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml index 5b6c53a0261..a7c061302b4 100644 --- a/solr/core/ivy.xml +++ b/solr/core/ivy.xml @@ -38,7 +38,6 @@ - @@ -110,6 +109,7 @@ + From d64c5c20b6de78f71ffb79f0249dd8a943d7c839 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Mon, 16 Dec 2019 08:04:33 +1100 Subject: [PATCH 13/15] SOLR-14087: disable package store API if -Denable.packages not set to true --- .../java/org/apache/solr/filestore/PackageStoreAPI.java | 6 +++++- solr/core/src/java/org/apache/solr/pkg/PackageAPI.java | 8 +++++++- .../apache/solr/filestore/TestDistribPackageStore.java | 8 ++++---- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java index f1750eafe8c..5c09d5f22d4 100644 --- a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java +++ b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java @@ -47,6 +47,7 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.core.BlobRepository; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; +import org.apache.solr.pkg.PackageAPI; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.PermissionNameProvider; @@ -64,7 +65,7 @@ import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM; public class PackageStoreAPI { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String PACKAGESTORE_DIRECTORY = "filestore"; + public static final String PACKAGESTORE_DIRECTORY = "$filestore"; private final CoreContainer coreContainer; @@ -135,6 +136,9 @@ public class PackageStoreAPI { @Command public void upload(SolrQueryRequest req, SolrQueryResponse rsp) { + if(!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { + throw new RuntimeException(PackageAPI.ERR_MSG); + } try { coreContainer.getZkController().getZkClient().create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index be4e9c9b612..1b93d894428 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -64,6 +64,8 @@ public class PackageAPI { public final boolean enablePackages = Boolean.parseBoolean(System.getProperty("enable.packages", "false")); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final String ERR_MSG = "Package loading is not enabled , Start your nodes with -Denable.packages=true"; + final CoreContainer coreContainer; private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); private final PackageLoader packageLoader; @@ -341,9 +343,13 @@ public class PackageAPI { } + public boolean isEnabled() { + return enablePackages; + } + private boolean checkEnabled(CommandOperation payload) { if (!enablePackages) { - payload.addError("Package loading is not enabled , Start your nodes with -Denable.packages=true"); + payload.addError(ERR_MSG); return false; } return true; diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java index bd4b6bd99eb..64b1ef8e153 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java @@ -17,9 +17,6 @@ package org.apache.solr.filestore; -import static org.apache.solr.common.util.Utils.JAVABINCONSUMER; -import static org.apache.solr.core.TestDynamicLoading.getFileContent; - import java.io.FileInputStream; import java.io.IOException; import java.nio.ByteBuffer; @@ -30,6 +27,7 @@ import java.util.Set; import java.util.concurrent.Callable; import java.util.function.Predicate; +import com.google.common.collect.ImmutableSet; import org.apache.commons.codec.digest.DigestUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; @@ -50,12 +48,14 @@ import org.apache.solr.util.LogLevel; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.server.ByteBufferInputStream; -import com.google.common.collect.ImmutableSet; +import static org.apache.solr.common.util.Utils.JAVABINCONSUMER; +import static org.apache.solr.core.TestDynamicLoading.getFileContent; @LogLevel("org.apache.solr.filestore.PackageStoreAPI=DEBUG;org.apache.solr.filestore.DistribPackageStore=DEBUG") public class TestDistribPackageStore extends SolrCloudTestCase { public void testPackageStoreManagement() throws Exception { + System.setProperty("enable.packages", "true"); MiniSolrCloudCluster cluster = configureCluster(4) .withJettyConfig(jetty -> jetty.enableV2(true)) From 479db61df82226e26ce68ce67e28fc51bce0b06a Mon Sep 17 00:00:00 2001 From: noble Date: Mon, 16 Dec 2019 15:49:54 +1100 Subject: [PATCH 14/15] SOLR-14087: changed the filestore location to .filestore instead of $filestore --- .../src/java/org/apache/solr/filestore/PackageStoreAPI.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java index 5c09d5f22d4..45cb01fe340 100644 --- a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java +++ b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java @@ -65,7 +65,7 @@ import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM; public class PackageStoreAPI { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String PACKAGESTORE_DIRECTORY = "$filestore"; + public static final String PACKAGESTORE_DIRECTORY = ".filestore"; private final CoreContainer coreContainer; From 2db48314f97a6017fe216f77c34cf5b261985cf4 Mon Sep 17 00:00:00 2001 From: David Smiley Date: Sun, 15 Dec 2019 23:53:17 -0500 Subject: [PATCH 15/15] SOLR-14072: Deprecate Blob API and runtimeLib (#1086) --- solr/CHANGES.txt | 4 ++++ .../src/adding-custom-plugins-in-solrcloud-mode.adoc | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 14a17a688d2..9f7e3daf2dd 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -183,6 +183,10 @@ Upgrade Notes * SOLR-14065: VelocityResponseWriter has been deprecated and may be removed in a future version. +* SOLR-14072: The "Blob Store" API and "runtimeLib" plugin mechanism that uses it is now considered deprecated. The + replacement to it is the "Package Management" system, which includes a "File Store". These are experimental currently + but will grow/stabalize/mature. + New Features --------------------- * SOLR-13821: A Package store to store and load package artifacts (noble, Ishan Chattopadhyaya) diff --git a/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc b/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc index 95756f7bb7e..bafc6e6a05e 100644 --- a/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc +++ b/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc @@ -18,6 +18,12 @@ In SolrCloud mode, custom plugins need to be shared across all nodes of the cluster. +.Deprecated +[IMPORTANT] +==== +The functionality here is a subset of the <> system. It will no longer be supported in Solr 9. +==== + When running Solr in SolrCloud mode and you want to use custom code (such as custom analyzers, tokenizers, query parsers, and other plugins), it can be cumbersome to add jars to the classpath on all nodes in your cluster. Using the <> and special commands with the <>, you can upload jars to a special system-level collection and dynamically load plugins from them at runtime without needing to restart any nodes. .This Feature is Disabled By Default