diff --git a/gradle/solr/packaging.gradle b/gradle/solr/packaging.gradle
index 3b1ea92d7ed..39b4bc68ebc 100644
--- a/gradle/solr/packaging.gradle
+++ b/gradle/solr/packaging.gradle
@@ -76,7 +76,8 @@ configure(allprojects.findAll {project -> project.path.startsWith(":solr:contrib
return true
}
}
- return externalLibs - configurations.solrPlatformLibs
+ // libExt has logging libs, which we don't want. Lets users decide what they want.
+ return externalLibs - configurations.solrPlatformLibs - project(':solr:server').configurations.getByName('libExt')
}, {
exclude "lucene-*"
into "lib"
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f1eb0c07c70..a379b1e5d11 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -42,8 +42,6 @@ Improvements
* SOLR-14880: Support coreRootDirectory setting when create new cores from command line, in standalone mode (Alexandre Rafalovitch)
-* SOLR-14972: Change default port of prometheus exporter to 8989 because it clashed with default embedded zookeeper port (janhoy)
-
* SOLR-14926, SOLR-14926, SOLR-13506: Modernize and clean up search results clustering contrib. This issue upgrades
the clustering contrib to the new Carrot2 4.x line, dropping several CVE-prone dependencies along the way.
The parameters and configuration of the contrib extensions have changed. The documentation in Solr ref guide
diff --git a/solr/contrib/prometheus-exporter/CHANGES.md b/solr/contrib/prometheus-exporter/CHANGES.md
new file mode 100644
index 00000000000..42deac6baec
--- /dev/null
+++ b/solr/contrib/prometheus-exporter/CHANGES.md
@@ -0,0 +1,20 @@
+This file lists release notes for this module.
+Prior to version 9, changes were in Solr's CHANGES.txt
+
+9.0.0
+======================
+
+Improvements
+----------------------
+* SOLR-14972: Change default port of prometheus exporter to 8989
+ because it clashed with default embedded zookeeper port (janhoy)
+
+Other Changes
+----------------------
+* SOLR-14915: Reduced dependencies from Solr server down to just SolrJ. Don't add WEB-INF/lib.
+ * Can run via gradle, "gradlew run"
+ * Has own log4j2.xml now
+ * Was missing some dependencies in lib/; now has all except SolrJ & logging.
+ (David Smiley, Houston Putman)
+
+* SOLR-14957: Add Prometheus Exporter to docker PATH. Fix classpath issues. (Houston Putman)
diff --git a/solr/contrib/prometheus-exporter/bin/solr-exporter b/solr/contrib/prometheus-exporter/bin/solr-exporter
index b601a785117..e53c46149db 100755
--- a/solr/contrib/prometheus-exporter/bin/solr-exporter
+++ b/solr/contrib/prometheus-exporter/bin/solr-exporter
@@ -82,10 +82,6 @@ for JAR in $(find "$BASEDIR"/../../dist/solrj-lib -name '*.jar')
do
CLASSPATH="$CLASSPATH":"$JAR"
done
-for JAR in $(find "$BASEDIR"/../../dist -name 'solr-core-*.jar')
-do
- CLASSPATH="$CLASSPATH":"$JAR"
-done
for JAR in $(find "$BASEDIR"/../../dist -name 'solr-solrj-*.jar')
do
CLASSPATH="$CLASSPATH":"$JAR"
@@ -94,14 +90,6 @@ for JAR in $(find "$BASEDIR"/../../dist -name 'solr-prometheus-exporter-*.jar')
do
CLASSPATH="$CLASSPATH":"$JAR"
done
-for JAR in $(find "$BASEDIR"/lucene-libs -name '*.jar')
-do
- CLASSPATH="$CLASSPATH":"$JAR"
-done
-for JAR in $(find "$BASEDIR"/../../server/solr-webapp/webapp/WEB-INF/lib -name '*.jar')
-do
- CLASSPATH="$CLASSPATH":"$JAR"
-done
for JAR in $(find "$BASEDIR"/../../server/lib/ext -name '*.jar')
do
CLASSPATH="$CLASSPATH":"$JAR"
@@ -123,8 +111,6 @@ else
GC_TUNE="$GC_TUNE"
fi
-EXTRA_JVM_ARGUMENTS="-Dlog4j.configurationFile=file:"$BASEDIR"/../../server/resources/log4j2-console.xml"
-
# For Cygwin, switch paths to Windows format before running java
if $cygwin; then
[ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
@@ -163,7 +149,6 @@ exec "$JAVACMD" \
$JAVA_MEM_OPTS \
$GC_TUNE \
$JAVA_OPTS \
- $EXTRA_JVM_ARGUMENTS \
$ZK_CREDS_AND_ACLS \
-classpath "$CLASSPATH" \
-Dapp.name="solr-exporter" \
diff --git a/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd b/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
index efa92a1608e..113306a84ee 100644
--- a/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
+++ b/solr/contrib/prometheus-exporter/bin/solr-exporter.cmd
@@ -71,8 +71,7 @@ if "%JAVACMD%"=="" set JAVACMD=java
if "%REPO%"=="" set REPO=%BASEDIR%\lib
-set CLASSPATH=%REPO%\*;%BASEDIR%\conf;%BASEDIR%\..\..\dist\solrj-lib\*;%BASEDIR%\..\..\dist\*;%BASEDIR%\lucene-libs\*;%BASEDIR%\..\..\server\solr-webapp\webapp\WEB-INF\lib\*;%BASEDIR%\..\..\server\lib\ext\*
-set EXTRA_JVM_ARGUMENTS=-Dlog4j.configurationFile=file:///%BASEDIR%\..\..\server\resources\log4j2-console.xml
+set CLASSPATH=%REPO%\*;%BASEDIR%\conf;%BASEDIR%\..\..\dist\solrj-lib\*;%BASEDIR%\..\..\dist\*;%BASEDIR%\..\..\server\lib\ext\*
@REM Convert Environment Variables to Command Line Options
set EXPORTER_ARGS=
@@ -88,7 +87,7 @@ goto endInit
@REM Reaching here means variables are defined and arguments have been captured
:endInit
-%JAVACMD% %JAVA_MEM% %GC_TUNE% %JAVA_OPTS% %EXTRA_JVM_ARGUMENTS% %ZK_CREDS_AND_ACLS% -classpath "%CLASSPATH_PREFIX%;%CLASSPATH%" -Dapp.name="solr-exporter" -Dapp.repo="%REPO%" -Dbasedir="%BASEDIR%" org.apache.solr.prometheus.exporter.SolrExporter %EXPORTER_ARGS% %CMD_LINE_ARGS%
+%JAVACMD% %JAVA_MEM% %GC_TUNE% %JAVA_OPTS% %ZK_CREDS_AND_ACLS% -classpath "%CLASSPATH_PREFIX%;%CLASSPATH%" -Dapp.name="solr-exporter" -Dapp.repo="%REPO%" -Dbasedir="%BASEDIR%" org.apache.solr.prometheus.exporter.SolrExporter %EXPORTER_ARGS% %CMD_LINE_ARGS%
if ERRORLEVEL 1 goto error
goto end
diff --git a/solr/contrib/prometheus-exporter/build.gradle b/solr/contrib/prometheus-exporter/build.gradle
index 13a9748bc44..ca4f625d073 100644
--- a/solr/contrib/prometheus-exporter/build.gradle
+++ b/solr/contrib/prometheus-exporter/build.gradle
@@ -15,14 +15,13 @@
* limitations under the License.
*/
-
apply plugin: 'java-library'
+// this is actually more of an 'application' but we don't want all of what Gradle adds
description = 'Prometheus exporter for exposing metrics from Solr using Metrics API and Search API'
dependencies {
- implementation project(':solr:core')
- implementation project(':lucene:analysis:common')
+ implementation project(':solr:solrj')
implementation ('io.prometheus:simpleclient')
implementation ('io.prometheus:simpleclient_common')
@@ -31,17 +30,57 @@ dependencies {
exclude group: "org.jruby.joni", module: "joni"
})
implementation ('net.sourceforge.argparse4j:argparse4j')
+ implementation ('com.github.ben-manes.caffeine:caffeine', {
+ exclude group: "org.checkerframework", module: "checker-qual"
+ exclude group: "com.google.errorprone", module: "error_prone_annotations"
+ })
- testImplementation ('org.apache.httpcomponents:httpcore')
- testImplementation ('org.eclipse.jetty:jetty-servlet')
+ runtimeOnly 'org.apache.logging.log4j:log4j-api'
+ runtimeOnly 'org.apache.logging.log4j:log4j-core'
+ runtimeOnly 'org.apache.logging.log4j:log4j-slf4j-impl'
+ runtimeOnly 'com.lmax:disruptor'
testImplementation project(':solr:test-framework')
}
-// Add two folders to default packaging.
+ext {
+ mainClass = 'org.apache.solr.prometheus.exporter.SolrExporter'
+}
+
+task run(type: JavaExec) {
+ group = 'application'
+ description = 'Run the main class with JavaExecTask'
+ main = project.ext.mainClass
+ classpath = sourceSets.main.runtimeClasspath
+ systemProperties = ["log4j.configurationFile":"file:conf/log4j2.xml"]
+}
+
+jar {
+ manifest {
+ attributes('Main-Class': project.ext.mainClass)
+ }
+}
+
assemblePackaging {
+ // Add two folders to default packaging.
from(projectDir, {
include "bin/**"
include "conf/**"
})
+ // Add all libs except those provided by SolrJ & Logging
+ from ({
+ def externalLibs = configurations.runtimeLibs.copyRecursive { dep ->
+ if (dep instanceof org.gradle.api.artifacts.ProjectDependency) {
+ return !dep.dependencyProject.path.startsWith(":solr")
+ } else {
+ return true
+ }
+ }
+ return externalLibs - project(':solr:server').configurations.getByName('libExt')
+ }, {
+ into "lib"
+ })
+
+
+ into deps
}
\ No newline at end of file
diff --git a/solr/contrib/prometheus-exporter/conf/log4j2.xml b/solr/contrib/prometheus-exporter/conf/log4j2.xml
new file mode 100644
index 00000000000..47f533f1054
--- /dev/null
+++ b/solr/contrib/prometheus-exporter/conf/log4j2.xml
@@ -0,0 +1,40 @@
+
+
+
+
+
+
+
+
+
+
+ %maxLen{%-5p - %d{yyyy-MM-dd HH:mm:ss.SSS}; %c; %m%notEmpty{ =>%ex{short}}}{10240}%n
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsConfiguration.java b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsConfiguration.java
index f976f3e564d..a4e630dbcbf 100644
--- a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsConfiguration.java
+++ b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsConfiguration.java
@@ -17,14 +17,27 @@
package org.apache.solr.prometheus.exporter;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathFactory;
+import java.io.File;
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
import net.thisptr.jackson.jq.exception.JsonQueryException;
-import org.apache.solr.core.XmlConfigFile;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
public class MetricsConfiguration {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final PrometheusExporterSettings settings;
@@ -66,13 +79,36 @@ public class MetricsConfiguration {
return searchConfiguration;
}
- public static MetricsConfiguration from(XmlConfigFile config) throws Exception {
- Node settings = config.getNode("/config/settings", false);
+ public static MetricsConfiguration from(String resource) throws Exception {
+ // See solr-core XmlConfigFile
+ final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ try {
+ dbf.setXIncludeAware(true);
+ dbf.setNamespaceAware(true);
+ } catch (UnsupportedOperationException e) {
+ log.warn("{} XML parser doesn't support XInclude option", resource);
+ }
- Node pingConfig = config.getNode("/config/rules/ping", false);
- Node metricsConfig = config.getNode("/config/rules/metrics", false);
- Node collectionsConfig = config.getNode("/config/rules/collections", false);
- Node searchConfiguration = config.getNode("/config/rules/search", false);
+ Document document;
+ Path path = Path.of(resource);
+ if (Files.exists(path)) {
+ document = dbf.newDocumentBuilder().parse(path.toUri().toASCIIString());
+ } else {
+ try (InputStream configInputStream = MethodHandles.lookup().lookupClass().getClassLoader().getResourceAsStream(resource.replace(File.separatorChar, '/'))) {
+ document = dbf.newDocumentBuilder().parse(configInputStream);
+ }
+ }
+
+ return from(document);
+ }
+
+ public static MetricsConfiguration from(Document config) throws Exception {
+ Node settings = getNode(config, "/config/settings");
+
+ Node pingConfig = getNode(config, "/config/rules/ping");
+ Node metricsConfig = getNode(config, "/config/rules/metrics");
+ Node collectionsConfig = getNode(config, "/config/rules/collections");
+ Node searchConfiguration = getNode(config, "/config/rules/search");
return new MetricsConfiguration(
settings == null ? PrometheusExporterSettings.builder().build() : PrometheusExporterSettings.from(settings),
@@ -83,6 +119,28 @@ public class MetricsConfiguration {
);
}
+ static final XPathFactory xpathFactory = XPathFactory.newInstance();
+
+ private static Node getNode(Document doc, String path) {
+ // Copied from solr-core XmlConfigFile.getNode with simplifications
+ XPath xpath = xpathFactory.newXPath();
+ String xstr = path; //normalize(path);
+
+ try {
+ NodeList nodes = (NodeList) xpath.evaluate(xstr, doc,
+ XPathConstants.NODESET);
+ if (nodes == null || 0 == nodes.getLength()) {
+ return null;
+ }
+ if (1 < nodes.getLength()) {
+ throw new RuntimeException("more than one value");
+ }
+ return nodes.item(0);
+ } catch (Exception e) {
+ throw new RuntimeException("Error in xpath:" + xstr, e);
+ }
+ }
+
private static List toMetricQueries(Node node) throws JsonQueryException {
if (node == null) {
return Collections.emptyList();
diff --git a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java
index d3da263fb77..b4b9f565201 100644
--- a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java
+++ b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java
@@ -19,8 +19,6 @@ package org.apache.solr.prometheus.exporter;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.InetSocketAddress;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.Locale;
import java.util.concurrent.ExecutorService;
@@ -32,8 +30,6 @@ import net.sourceforge.argparse4j.inf.ArgumentParserException;
import net.sourceforge.argparse4j.inf.Namespace;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.core.XmlConfigFile;
import org.apache.solr.prometheus.collector.MetricsCollectorFactory;
import org.apache.solr.prometheus.collector.SchedulerMetricsCollector;
import org.apache.solr.prometheus.scraper.SolrCloudScraper;
@@ -202,7 +198,7 @@ public class SolrExporter {
res.getInt(ARG_NUM_THREADS_DEST),
res.getInt(ARG_SCRAPE_INTERVAL_DEST),
scrapeConfiguration,
- loadMetricsConfiguration(Paths.get(res.getString(ARG_CONFIG_DEST))));
+ loadMetricsConfiguration(res.getString(ARG_CONFIG_DEST)));
log.info("Starting Solr Prometheus Exporting");
solrExporter.start();
@@ -214,12 +210,11 @@ public class SolrExporter {
}
}
- private static MetricsConfiguration loadMetricsConfiguration(Path configPath) {
- try (SolrResourceLoader loader = new SolrResourceLoader(configPath.getParent())) {
- XmlConfigFile config = new XmlConfigFile(loader, configPath.getFileName().toString(), null, null);
- return MetricsConfiguration.from(config);
+ private static MetricsConfiguration loadMetricsConfiguration(String configPath) {
+ try {
+ return MetricsConfiguration.from(configPath);
} catch (Exception e) {
- log.error("Could not load scrape configuration from {}", configPath.toAbsolutePath());
+ log.error("Could not load scrape configuration from {}", configPath);
throw new RuntimeException(e);
}
}
diff --git a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
index e4b98e75811..5efedc36af7 100644
--- a/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
+++ b/solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java
@@ -20,13 +20,12 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.function.Function;
import java.util.stream.Collectors;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.cloud.DocCollection;
@@ -42,7 +41,7 @@ public class SolrCloudScraper extends SolrScraper {
private final CloudSolrClient solrClient;
private final SolrClientFactory solrClientFactory;
- private Cache hostClientCache = CacheBuilder.newBuilder().build();
+ private Cache hostClientCache = Caffeine.newBuilder().build();
public SolrCloudScraper(CloudSolrClient solrClient, ExecutorService executor, SolrClientFactory solrClientFactory) {
super(executor);
@@ -83,15 +82,8 @@ public class SolrCloudScraper extends SolrScraper {
private Map createHttpSolrClients() throws IOException {
return getBaseUrls().stream()
- .map(url -> {
- try {
- return hostClientCache.get(url, () -> solrClientFactory.createStandaloneSolrClient(url));
- } catch (ExecutionException e) {
- throw new RuntimeException(e);
- }
- })
+ .map(url -> hostClientCache.get(url, solrClientFactory::createStandaloneSolrClient))
.collect(Collectors.toMap(HttpSolrClient::getBaseURL, Function.identity()));
-
}
@Override
diff --git a/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/utils/Helpers.java b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/utils/Helpers.java
index 5dcfb715335..6870cb2aacc 100644
--- a/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/utils/Helpers.java
+++ b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/utils/Helpers.java
@@ -19,28 +19,19 @@ package org.apache.solr.prometheus.utils;
import java.io.File;
import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.Objects;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.core.XmlConfigFile;
import org.apache.solr.prometheus.PrometheusExporterTestBase;
import org.apache.solr.prometheus.exporter.MetricsConfiguration;
public class Helpers {
- public static MetricsConfiguration loadConfiguration(String path) throws Exception {
- Path configPath = Paths.get(path);
-
- try (SolrResourceLoader loader = new SolrResourceLoader(configPath.getParent())) {
- XmlConfigFile config = new XmlConfigFile(loader, configPath.getFileName().toString());
- return MetricsConfiguration.from(config);
- }
+ public static MetricsConfiguration loadConfiguration(String pathRsrc) throws Exception {
+ return MetricsConfiguration.from(SolrTestCaseJ4.getFile(pathRsrc).getPath());
}
public static void indexAllDocs(SolrClient client) throws IOException, SolrServerException {