mirror of https://github.com/apache/lucene.git
SOLR-14915: Prometheus-exporter should not depend on Solr-core (#1972)
* Reduced dependencies from Solr server down to just SolrJ. Don't add WEB-INF/lib. * Was missing some dependencies in lib/; now has all except SolrJ & logging. * Can run via gradle, "gradlew run" * Has own log4j2.xml now Has own CHANGES.md now.
This commit is contained in:
David Smiley
GitHub
parent
1e0ae2fb74
commit
021de9f45f
|
|
@ -76,7 +76,8 @@ configure(allprojects.findAll {project -> project.path.startsWith(":solr:contrib
|
|||
return true
|
||||
}
|
||||
}
|
||||
return externalLibs - configurations.solrPlatformLibs
|
||||
// libExt has logging libs, which we don't want. Lets users decide what they want.
|
||||
return externalLibs - configurations.solrPlatformLibs - project(':solr:server').configurations.getByName('libExt')
|
||||
}, {
|
||||
exclude "lucene-*"
|
||||
into "lib"
|
||||
|
|
|
|||
|
|
@ -42,8 +42,6 @@ Improvements
|
|||
|
||||
* SOLR-14880: Support coreRootDirectory setting when create new cores from command line, in standalone mode (Alexandre Rafalovitch)
|
||||
|
||||
* SOLR-14972: Change default port of prometheus exporter to 8989 because it clashed with default embedded zookeeper port (janhoy)
|
||||
|
||||
* SOLR-14926, SOLR-14926, SOLR-13506: Modernize and clean up search results clustering contrib. This issue upgrades
|
||||
the clustering contrib to the new Carrot2 4.x line, dropping several CVE-prone dependencies along the way.
|
||||
The parameters and configuration of the contrib extensions have changed. The documentation in Solr ref guide
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
This file lists release notes for this module.
|
||||
Prior to version 9, changes were in Solr's CHANGES.txt
|
||||
|
||||
9.0.0
|
||||
======================
|
||||
|
||||
Improvements
|
||||
----------------------
|
||||
* SOLR-14972: Change default port of prometheus exporter to 8989
|
||||
because it clashed with default embedded zookeeper port (janhoy)
|
||||
|
||||
Other Changes
|
||||
----------------------
|
||||
* SOLR-14915: Reduced dependencies from Solr server down to just SolrJ. Don't add WEB-INF/lib.
|
||||
* Can run via gradle, "gradlew run"
|
||||
* Has own log4j2.xml now
|
||||
* Was missing some dependencies in lib/; now has all except SolrJ & logging.
|
||||
(David Smiley, Houston Putman)
|
||||
|
||||
* SOLR-14957: Add Prometheus Exporter to docker PATH. Fix classpath issues. (Houston Putman)
|
||||
|
|
@ -82,10 +82,6 @@ for JAR in $(find "$BASEDIR"/../../dist/solrj-lib -name '*.jar')
|
|||
do
|
||||
CLASSPATH="$CLASSPATH":"$JAR"
|
||||
done
|
||||
for JAR in $(find "$BASEDIR"/../../dist -name 'solr-core-*.jar')
|
||||
do
|
||||
CLASSPATH="$CLASSPATH":"$JAR"
|
||||
done
|
||||
for JAR in $(find "$BASEDIR"/../../dist -name 'solr-solrj-*.jar')
|
||||
do
|
||||
CLASSPATH="$CLASSPATH":"$JAR"
|
||||
|
|
@ -94,14 +90,6 @@ for JAR in $(find "$BASEDIR"/../../dist -name 'solr-prometheus-exporter-*.jar')
|
|||
do
|
||||
CLASSPATH="$CLASSPATH":"$JAR"
|
||||
done
|
||||
for JAR in $(find "$BASEDIR"/lucene-libs -name '*.jar')
|
||||
do
|
||||
CLASSPATH="$CLASSPATH":"$JAR"
|
||||
done
|
||||
for JAR in $(find "$BASEDIR"/../../server/solr-webapp/webapp/WEB-INF/lib -name '*.jar')
|
||||
do
|
||||
CLASSPATH="$CLASSPATH":"$JAR"
|
||||
done
|
||||
for JAR in $(find "$BASEDIR"/../../server/lib/ext -name '*.jar')
|
||||
do
|
||||
CLASSPATH="$CLASSPATH":"$JAR"
|
||||
|
|
@ -123,8 +111,6 @@ else
|
|||
GC_TUNE="$GC_TUNE"
|
||||
fi
|
||||
|
||||
EXTRA_JVM_ARGUMENTS="-Dlog4j.configurationFile=file:"$BASEDIR"/../../server/resources/log4j2-console.xml"
|
||||
|
||||
# For Cygwin, switch paths to Windows format before running java
|
||||
if $cygwin; then
|
||||
[ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
|
||||
|
|
@ -163,7 +149,6 @@ exec "$JAVACMD" \
|
|||
$JAVA_MEM_OPTS \
|
||||
$GC_TUNE \
|
||||
$JAVA_OPTS \
|
||||
$EXTRA_JVM_ARGUMENTS \
|
||||
$ZK_CREDS_AND_ACLS \
|
||||
-classpath "$CLASSPATH" \
|
||||
-Dapp.name="solr-exporter" \
|
||||
|
|
|
|||
|
|
@ -71,8 +71,7 @@ if "%JAVACMD%"=="" set JAVACMD=java
|
|||
|
||||
if "%REPO%"=="" set REPO=%BASEDIR%\lib
|
||||
|
||||
set CLASSPATH=%REPO%\*;%BASEDIR%\conf;%BASEDIR%\..\..\dist\solrj-lib\*;%BASEDIR%\..\..\dist\*;%BASEDIR%\lucene-libs\*;%BASEDIR%\..\..\server\solr-webapp\webapp\WEB-INF\lib\*;%BASEDIR%\..\..\server\lib\ext\*
|
||||
set EXTRA_JVM_ARGUMENTS=-Dlog4j.configurationFile=file:///%BASEDIR%\..\..\server\resources\log4j2-console.xml
|
||||
set CLASSPATH=%REPO%\*;%BASEDIR%\conf;%BASEDIR%\..\..\dist\solrj-lib\*;%BASEDIR%\..\..\dist\*;%BASEDIR%\..\..\server\lib\ext\*
|
||||
|
||||
@REM Convert Environment Variables to Command Line Options
|
||||
set EXPORTER_ARGS=
|
||||
|
|
@ -88,7 +87,7 @@ goto endInit
|
|||
@REM Reaching here means variables are defined and arguments have been captured
|
||||
:endInit
|
||||
|
||||
%JAVACMD% %JAVA_MEM% %GC_TUNE% %JAVA_OPTS% %EXTRA_JVM_ARGUMENTS% %ZK_CREDS_AND_ACLS% -classpath "%CLASSPATH_PREFIX%;%CLASSPATH%" -Dapp.name="solr-exporter" -Dapp.repo="%REPO%" -Dbasedir="%BASEDIR%" org.apache.solr.prometheus.exporter.SolrExporter %EXPORTER_ARGS% %CMD_LINE_ARGS%
|
||||
%JAVACMD% %JAVA_MEM% %GC_TUNE% %JAVA_OPTS% %ZK_CREDS_AND_ACLS% -classpath "%CLASSPATH_PREFIX%;%CLASSPATH%" -Dapp.name="solr-exporter" -Dapp.repo="%REPO%" -Dbasedir="%BASEDIR%" org.apache.solr.prometheus.exporter.SolrExporter %EXPORTER_ARGS% %CMD_LINE_ARGS%
|
||||
if ERRORLEVEL 1 goto error
|
||||
goto end
|
||||
|
||||
|
|
|
|||
|
|
@ -15,14 +15,13 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
apply plugin: 'java-library'
|
||||
// this is actually more of an 'application' but we don't want all of what Gradle adds
|
||||
|
||||
description = 'Prometheus exporter for exposing metrics from Solr using Metrics API and Search API'
|
||||
|
||||
dependencies {
|
||||
implementation project(':solr:core')
|
||||
implementation project(':lucene:analysis:common')
|
||||
implementation project(':solr:solrj')
|
||||
|
||||
implementation ('io.prometheus:simpleclient')
|
||||
implementation ('io.prometheus:simpleclient_common')
|
||||
|
|
@ -31,17 +30,57 @@ dependencies {
|
|||
exclude group: "org.jruby.joni", module: "joni"
|
||||
})
|
||||
implementation ('net.sourceforge.argparse4j:argparse4j')
|
||||
implementation ('com.github.ben-manes.caffeine:caffeine', {
|
||||
exclude group: "org.checkerframework", module: "checker-qual"
|
||||
exclude group: "com.google.errorprone", module: "error_prone_annotations"
|
||||
})
|
||||
|
||||
testImplementation ('org.apache.httpcomponents:httpcore')
|
||||
testImplementation ('org.eclipse.jetty:jetty-servlet')
|
||||
runtimeOnly 'org.apache.logging.log4j:log4j-api'
|
||||
runtimeOnly 'org.apache.logging.log4j:log4j-core'
|
||||
runtimeOnly 'org.apache.logging.log4j:log4j-slf4j-impl'
|
||||
runtimeOnly 'com.lmax:disruptor'
|
||||
|
||||
testImplementation project(':solr:test-framework')
|
||||
}
|
||||
|
||||
// Add two folders to default packaging.
|
||||
ext {
|
||||
mainClass = 'org.apache.solr.prometheus.exporter.SolrExporter'
|
||||
}
|
||||
|
||||
task run(type: JavaExec) {
|
||||
group = 'application'
|
||||
description = 'Run the main class with JavaExecTask'
|
||||
main = project.ext.mainClass
|
||||
classpath = sourceSets.main.runtimeClasspath
|
||||
systemProperties = ["log4j.configurationFile":"file:conf/log4j2.xml"]
|
||||
}
|
||||
|
||||
jar {
|
||||
manifest {
|
||||
attributes('Main-Class': project.ext.mainClass)
|
||||
}
|
||||
}
|
||||
|
||||
assemblePackaging {
|
||||
// Add two folders to default packaging.
|
||||
from(projectDir, {
|
||||
include "bin/**"
|
||||
include "conf/**"
|
||||
})
|
||||
// Add all libs except those provided by SolrJ & Logging
|
||||
from ({
|
||||
def externalLibs = configurations.runtimeLibs.copyRecursive { dep ->
|
||||
if (dep instanceof org.gradle.api.artifacts.ProjectDependency) {
|
||||
return !dep.dependencyProject.path.startsWith(":solr")
|
||||
} else {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return externalLibs - project(':solr:server').configurations.getByName('libExt')
|
||||
}, {
|
||||
into "lib"
|
||||
})
|
||||
|
||||
|
||||
into deps
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
<!-- Use this file for logging exlusively to the console, useful for
|
||||
some development tasks. Should not be used for production -->
|
||||
<!-- Default production configuration is asnychronous logging -->
|
||||
<Configuration>
|
||||
<Appenders>
|
||||
<Console name="STDERR" target="SYSTEM_ERR">
|
||||
<PatternLayout>
|
||||
<Pattern>
|
||||
%maxLen{%-5p - %d{yyyy-MM-dd HH:mm:ss.SSS}; %c; %m%notEmpty{ =>%ex{short}}}{10240}%n
|
||||
</Pattern>
|
||||
</PatternLayout>
|
||||
</Console>
|
||||
</Appenders>
|
||||
<Loggers>
|
||||
<!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
|
||||
<AsyncRoot level="INFO">
|
||||
<AppenderRef ref="STDERR"/>
|
||||
</AsyncRoot>
|
||||
</Loggers>
|
||||
</Configuration>
|
||||
|
||||
|
||||
|
|
@ -17,14 +17,27 @@
|
|||
|
||||
package org.apache.solr.prometheus.exporter;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.xpath.XPath;
|
||||
import javax.xml.xpath.XPathConstants;
|
||||
import javax.xml.xpath.XPathFactory;
|
||||
import java.io.File;
|
||||
import java.io.InputStream;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import net.thisptr.jackson.jq.exception.JsonQueryException;
|
||||
import org.apache.solr.core.XmlConfigFile;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
|
||||
public class MetricsConfiguration {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
private final PrometheusExporterSettings settings;
|
||||
|
||||
|
|
@ -66,13 +79,36 @@ public class MetricsConfiguration {
|
|||
return searchConfiguration;
|
||||
}
|
||||
|
||||
public static MetricsConfiguration from(XmlConfigFile config) throws Exception {
|
||||
Node settings = config.getNode("/config/settings", false);
|
||||
public static MetricsConfiguration from(String resource) throws Exception {
|
||||
// See solr-core XmlConfigFile
|
||||
final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
||||
try {
|
||||
dbf.setXIncludeAware(true);
|
||||
dbf.setNamespaceAware(true);
|
||||
} catch (UnsupportedOperationException e) {
|
||||
log.warn("{} XML parser doesn't support XInclude option", resource);
|
||||
}
|
||||
|
||||
Node pingConfig = config.getNode("/config/rules/ping", false);
|
||||
Node metricsConfig = config.getNode("/config/rules/metrics", false);
|
||||
Node collectionsConfig = config.getNode("/config/rules/collections", false);
|
||||
Node searchConfiguration = config.getNode("/config/rules/search", false);
|
||||
Document document;
|
||||
Path path = Path.of(resource);
|
||||
if (Files.exists(path)) {
|
||||
document = dbf.newDocumentBuilder().parse(path.toUri().toASCIIString());
|
||||
} else {
|
||||
try (InputStream configInputStream = MethodHandles.lookup().lookupClass().getClassLoader().getResourceAsStream(resource.replace(File.separatorChar, '/'))) {
|
||||
document = dbf.newDocumentBuilder().parse(configInputStream);
|
||||
}
|
||||
}
|
||||
|
||||
return from(document);
|
||||
}
|
||||
|
||||
public static MetricsConfiguration from(Document config) throws Exception {
|
||||
Node settings = getNode(config, "/config/settings");
|
||||
|
||||
Node pingConfig = getNode(config, "/config/rules/ping");
|
||||
Node metricsConfig = getNode(config, "/config/rules/metrics");
|
||||
Node collectionsConfig = getNode(config, "/config/rules/collections");
|
||||
Node searchConfiguration = getNode(config, "/config/rules/search");
|
||||
|
||||
return new MetricsConfiguration(
|
||||
settings == null ? PrometheusExporterSettings.builder().build() : PrometheusExporterSettings.from(settings),
|
||||
|
|
@ -83,6 +119,28 @@ public class MetricsConfiguration {
|
|||
);
|
||||
}
|
||||
|
||||
static final XPathFactory xpathFactory = XPathFactory.newInstance();
|
||||
|
||||
private static Node getNode(Document doc, String path) {
|
||||
// Copied from solr-core XmlConfigFile.getNode with simplifications
|
||||
XPath xpath = xpathFactory.newXPath();
|
||||
String xstr = path; //normalize(path);
|
||||
|
||||
try {
|
||||
NodeList nodes = (NodeList) xpath.evaluate(xstr, doc,
|
||||
XPathConstants.NODESET);
|
||||
if (nodes == null || 0 == nodes.getLength()) {
|
||||
return null;
|
||||
}
|
||||
if (1 < nodes.getLength()) {
|
||||
throw new RuntimeException("more than one value");
|
||||
}
|
||||
return nodes.item(0);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Error in xpath:" + xstr, e);
|
||||
}
|
||||
}
|
||||
|
||||
private static List<MetricsQuery> toMetricQueries(Node node) throws JsonQueryException {
|
||||
if (node == null) {
|
||||
return Collections.emptyList();
|
||||
|
|
|
|||
|
|
@ -19,8 +19,6 @@ package org.apache.solr.prometheus.exporter;
|
|||
import java.io.IOException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Locale;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
|
||||
|
|
@ -32,8 +30,6 @@ import net.sourceforge.argparse4j.inf.ArgumentParserException;
|
|||
import net.sourceforge.argparse4j.inf.Namespace;
|
||||
import org.apache.solr.common.util.ExecutorUtil;
|
||||
import org.apache.solr.common.util.IOUtils;
|
||||
import org.apache.solr.core.SolrResourceLoader;
|
||||
import org.apache.solr.core.XmlConfigFile;
|
||||
import org.apache.solr.prometheus.collector.MetricsCollectorFactory;
|
||||
import org.apache.solr.prometheus.collector.SchedulerMetricsCollector;
|
||||
import org.apache.solr.prometheus.scraper.SolrCloudScraper;
|
||||
|
|
@ -202,7 +198,7 @@ public class SolrExporter {
|
|||
res.getInt(ARG_NUM_THREADS_DEST),
|
||||
res.getInt(ARG_SCRAPE_INTERVAL_DEST),
|
||||
scrapeConfiguration,
|
||||
loadMetricsConfiguration(Paths.get(res.getString(ARG_CONFIG_DEST))));
|
||||
loadMetricsConfiguration(res.getString(ARG_CONFIG_DEST)));
|
||||
|
||||
log.info("Starting Solr Prometheus Exporting");
|
||||
solrExporter.start();
|
||||
|
|
@ -214,12 +210,11 @@ public class SolrExporter {
|
|||
}
|
||||
}
|
||||
|
||||
private static MetricsConfiguration loadMetricsConfiguration(Path configPath) {
|
||||
try (SolrResourceLoader loader = new SolrResourceLoader(configPath.getParent())) {
|
||||
XmlConfigFile config = new XmlConfigFile(loader, configPath.getFileName().toString(), null, null);
|
||||
return MetricsConfiguration.from(config);
|
||||
private static MetricsConfiguration loadMetricsConfiguration(String configPath) {
|
||||
try {
|
||||
return MetricsConfiguration.from(configPath);
|
||||
} catch (Exception e) {
|
||||
log.error("Could not load scrape configuration from {}", configPath.toAbsolutePath());
|
||||
log.error("Could not load scrape configuration from {}", configPath);
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,13 +20,12 @@ import java.io.IOException;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.google.common.cache.Cache;
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.github.benmanes.caffeine.cache.Cache;
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||
import org.apache.solr.common.cloud.DocCollection;
|
||||
|
|
@ -42,7 +41,7 @@ public class SolrCloudScraper extends SolrScraper {
|
|||
private final CloudSolrClient solrClient;
|
||||
private final SolrClientFactory solrClientFactory;
|
||||
|
||||
private Cache<String, HttpSolrClient> hostClientCache = CacheBuilder.newBuilder().build();
|
||||
private Cache<String, HttpSolrClient> hostClientCache = Caffeine.newBuilder().build();
|
||||
|
||||
public SolrCloudScraper(CloudSolrClient solrClient, ExecutorService executor, SolrClientFactory solrClientFactory) {
|
||||
super(executor);
|
||||
|
|
@ -83,15 +82,8 @@ public class SolrCloudScraper extends SolrScraper {
|
|||
|
||||
private Map<String, HttpSolrClient> createHttpSolrClients() throws IOException {
|
||||
return getBaseUrls().stream()
|
||||
.map(url -> {
|
||||
try {
|
||||
return hostClientCache.get(url, () -> solrClientFactory.createStandaloneSolrClient(url));
|
||||
} catch (ExecutionException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
})
|
||||
.map(url -> hostClientCache.get(url, solrClientFactory::createStandaloneSolrClient))
|
||||
.collect(Collectors.toMap(HttpSolrClient::getBaseURL, Function.identity()));
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -19,28 +19,19 @@ package org.apache.solr.prometheus.utils;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
|
||||
import org.apache.solr.core.SolrResourceLoader;
|
||||
import org.apache.solr.core.XmlConfigFile;
|
||||
import org.apache.solr.prometheus.PrometheusExporterTestBase;
|
||||
import org.apache.solr.prometheus.exporter.MetricsConfiguration;
|
||||
|
||||
public class Helpers {
|
||||
|
||||
public static MetricsConfiguration loadConfiguration(String path) throws Exception {
|
||||
Path configPath = Paths.get(path);
|
||||
|
||||
try (SolrResourceLoader loader = new SolrResourceLoader(configPath.getParent())) {
|
||||
XmlConfigFile config = new XmlConfigFile(loader, configPath.getFileName().toString());
|
||||
return MetricsConfiguration.from(config);
|
||||
}
|
||||
public static MetricsConfiguration loadConfiguration(String pathRsrc) throws Exception {
|
||||
return MetricsConfiguration.from(SolrTestCaseJ4.getFile(pathRsrc).getPath());
|
||||
}
|
||||
|
||||
public static void indexAllDocs(SolrClient client) throws IOException, SolrServerException {
|
||||
|
|
|
|||
Loading…
Reference in New Issue