mirror of https://github.com/apache/lucene.git
SOLR-11795: Add Solr metrics exporter for Prometheus
This commit is contained in:
koji
parent
dfc0fe86e4
commit
4bfcbc5c60
|
|
@ -168,6 +168,8 @@ New Features
|
||||||
|
|
||||||
* SOLR-11588: Add matrixMult Stream Evaluator to support matrix multiplication (Joel Bernstein)
|
* SOLR-11588: Add matrixMult Stream Evaluator to support matrix multiplication (Joel Bernstein)
|
||||||
|
|
||||||
|
* SOLR-11795: Add Solr metrics exporter for Prometheus (Minoru Osuka via koji)
|
||||||
|
|
||||||
Bug Fixes
|
Bug Fixes
|
||||||
----------------------
|
----------------------
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -571,19 +571,19 @@
|
||||||
<tarfileset dir="."
|
<tarfileset dir="."
|
||||||
prefix="${fullnamever}"
|
prefix="${fullnamever}"
|
||||||
includes="LICENSE.txt NOTICE.txt CHANGES.txt README.txt SYSTEM_REQUIREMENTS.txt
|
includes="LICENSE.txt NOTICE.txt CHANGES.txt README.txt SYSTEM_REQUIREMENTS.txt
|
||||||
bin/** server/** example/** contrib/**/lib/** contrib/**/README.txt
|
bin/** server/** example/** contrib/**/lib/** contrib/**/conf/** contrib/**/README.txt
|
||||||
licenses/**"
|
licenses/**"
|
||||||
excludes="licenses/README.committers.txt **/data/ **/logs/*
|
excludes="licenses/README.committers.txt **/data/ **/logs/*
|
||||||
**/classes/ **/*.sh **/ivy.xml **/build.xml
|
**/classes/ **/*.sh **/ivy.xml **/build.xml
|
||||||
**/bin/ **/*.iml **/*.ipr **/*.iws **/pom.xml
|
**/bin/ **/*.iml **/*.ipr **/*.iws **/pom.xml
|
||||||
**/*pom.xml.template server/etc/test/" />
|
**/*pom.xml.template server/etc/test/ contrib/**/src/" />
|
||||||
<tarfileset dir="${dest}/contrib-lucene-libs-to-package"
|
<tarfileset dir="${dest}/contrib-lucene-libs-to-package"
|
||||||
prefix="${fullnamever}"
|
prefix="${fullnamever}"
|
||||||
includes="**" />
|
includes="**" />
|
||||||
<tarfileset dir="."
|
<tarfileset dir="."
|
||||||
filemode="755"
|
filemode="755"
|
||||||
prefix="${fullnamever}"
|
prefix="${fullnamever}"
|
||||||
includes="bin/** server/**/*.sh example/**/*.sh example/**/bin/"
|
includes="bin/** server/**/*.sh example/**/*.sh example/**/bin/ contrib/**/bin/**"
|
||||||
excludes="server/etc/test/**" />
|
excludes="server/etc/test/**" />
|
||||||
<tarfileset dir="."
|
<tarfileset dir="."
|
||||||
prefix="${fullnamever}"
|
prefix="${fullnamever}"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
Welcome to Apache Solr Prometheus Exporter
|
||||||
|
========
|
||||||
|
|
||||||
|
Apache Solr Prometheus Exporter (solr-exporter) provides a way for you to expose metrics for Solr to Prometheus.
|
||||||
|
|
||||||
|
# Getting Started With Solr Prometheus Exporter
|
||||||
|
|
||||||
|
For information on how to get started with solr-exporter please see:
|
||||||
|
* [Solr Reference Guide's section on Monitoring Solr with Prometheus and Grafana](https://lucene.apache.org/solr/guide/monitoring-solr-with-prometheus-and-grafana.html)
|
||||||
|
|
||||||
|
# Getting Started With Solr
|
||||||
|
|
||||||
|
For information on how to get started with solr please see:
|
||||||
|
* [solr/README.txt](../../README.txt)
|
||||||
|
* [Solr Tutorial](https://lucene.apache.org/solr/guide/solr-tutorial.html)
|
||||||
|
|
||||||
|
# How To Contribute
|
||||||
|
|
||||||
|
For information on how to contribute see:
|
||||||
|
* http://wiki.apache.org/lucene-java/HowToContribute
|
||||||
|
* http://wiki.apache.org/solr/HowToContribute
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
README.md
|
||||||
|
|
@ -0,0 +1,114 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
#
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
BASEDIR=`dirname $0`/..
|
||||||
|
BASEDIR=`(cd "$BASEDIR"; pwd)`
|
||||||
|
|
||||||
|
# OS specific support. $var _must_ be set to either true or false.
|
||||||
|
cygwin=false;
|
||||||
|
darwin=false;
|
||||||
|
case "`uname`" in
|
||||||
|
CYGWIN*) cygwin=true ;;
|
||||||
|
Darwin*) darwin=true
|
||||||
|
if [ -z "$JAVA_VERSION" ] ; then
|
||||||
|
JAVA_VERSION="CurrentJDK"
|
||||||
|
else
|
||||||
|
echo "Using Java version: $JAVA_VERSION"
|
||||||
|
fi
|
||||||
|
if [ -z "$JAVA_HOME" ] ; then
|
||||||
|
JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/${JAVA_VERSION}/Home
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
if [ -z "$JAVA_HOME" ] ; then
|
||||||
|
if [ -r /etc/gentoo-release ] ; then
|
||||||
|
JAVA_HOME=`java-config --jre-home`
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# For Cygwin, ensure paths are in UNIX format before anything is touched
|
||||||
|
if $cygwin ; then
|
||||||
|
[ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
|
||||||
|
[ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
|
||||||
|
fi
|
||||||
|
|
||||||
|
# If a specific java binary isn't specified search for the standard 'java' binary
|
||||||
|
if [ -z "$JAVACMD" ] ; then
|
||||||
|
if [ -n "$JAVA_HOME" ] ; then
|
||||||
|
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
|
||||||
|
# IBM's JDK on AIX uses strange locations for the executables
|
||||||
|
JAVACMD="$JAVA_HOME/jre/sh/java"
|
||||||
|
else
|
||||||
|
JAVACMD="$JAVA_HOME/bin/java"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
JAVACMD=`which java`
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -x "$JAVACMD" ] ; then
|
||||||
|
echo "Error: JAVA_HOME is not defined correctly."
|
||||||
|
echo " We cannot execute $JAVACMD"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "$REPO" ]
|
||||||
|
then
|
||||||
|
REPO="$BASEDIR"/lib
|
||||||
|
fi
|
||||||
|
|
||||||
|
CLASSPATH=$CLASSPATH_PREFIX
|
||||||
|
for JAR in $(find "$REPO" -name '*.jar')
|
||||||
|
do
|
||||||
|
CLASSPATH="$CLASSPATH":"$JAR"
|
||||||
|
done
|
||||||
|
for JAR in $(find "$BASEDIR"/../../dist/solrj-lib -name '*.jar')
|
||||||
|
do
|
||||||
|
CLASSPATH="$CLASSPATH":"$JAR"
|
||||||
|
done
|
||||||
|
for JAR in $(find "$BASEDIR"/../../dist -name 'solr-solrj-*.jar')
|
||||||
|
do
|
||||||
|
CLASSPATH="$CLASSPATH":"$JAR"
|
||||||
|
done
|
||||||
|
for JAR in $(find "$BASEDIR"/../../dist -name 'solr-prometheus-exporter-*.jar')
|
||||||
|
do
|
||||||
|
CLASSPATH="$CLASSPATH":"$JAR"
|
||||||
|
done
|
||||||
|
|
||||||
|
EXTRA_JVM_ARGUMENTS="-Xmx512m -Dlog4j.configuration=file:"$BASEDIR"/conf/log4j.properties"
|
||||||
|
|
||||||
|
# For Cygwin, switch paths to Windows format before running java
|
||||||
|
if $cygwin; then
|
||||||
|
[ -n "$CLASSPATH" ] && CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
|
||||||
|
[ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
|
||||||
|
[ -n "$HOME" ] && HOME=`cygpath --path --windows "$HOME"`
|
||||||
|
[ -n "$BASEDIR" ] && BASEDIR=`cygpath --path --windows "$BASEDIR"`
|
||||||
|
[ -n "$REPO" ] && REPO=`cygpath --path --windows "$REPO"`
|
||||||
|
fi
|
||||||
|
|
||||||
|
exec "$JAVACMD" $JAVA_OPTS \
|
||||||
|
$EXTRA_JVM_ARGUMENTS \
|
||||||
|
-classpath "$CLASSPATH" \
|
||||||
|
-Dapp.name="solr-exporter" \
|
||||||
|
-Dapp.pid="$$" \
|
||||||
|
-Dapp.repo="$REPO" \
|
||||||
|
-Dbasedir="$BASEDIR" \
|
||||||
|
org.apache.solr.prometheus.exporter.SolrExporter \
|
||||||
|
"$@"
|
||||||
|
|
@ -0,0 +1,105 @@
|
||||||
|
@REM
|
||||||
|
@REM Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
@REM contributor license agreements. See the NOTICE file distributed with
|
||||||
|
@REM this work for additional information regarding copyright ownership.
|
||||||
|
@REM The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
@REM (the "License"); you may not use this file except in compliance with
|
||||||
|
@REM the License. You may obtain a copy of the License at
|
||||||
|
@REM
|
||||||
|
@REM http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
@REM
|
||||||
|
@REM Unless required by applicable law or agreed to in writing, software
|
||||||
|
@REM distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
@REM See the License for the specific language governing permissions and
|
||||||
|
@REM limitations under the License.
|
||||||
|
@REM
|
||||||
|
|
||||||
|
@echo off
|
||||||
|
|
||||||
|
set ERROR_CODE=0
|
||||||
|
|
||||||
|
:init
|
||||||
|
@REM Decide how to startup depending on the version of windows
|
||||||
|
|
||||||
|
@REM -- Win98ME
|
||||||
|
if NOT "%OS%"=="Windows_NT" goto Win9xArg
|
||||||
|
|
||||||
|
@REM set local scope for the variables with windows NT shell
|
||||||
|
if "%OS%"=="Windows_NT" @setlocal
|
||||||
|
|
||||||
|
@REM -- 4NT shell
|
||||||
|
if "%eval[2+2]" == "4" goto 4NTArgs
|
||||||
|
|
||||||
|
@REM -- Regular WinNT shell
|
||||||
|
set CMD_LINE_ARGS=%*
|
||||||
|
goto WinNTGetScriptDir
|
||||||
|
|
||||||
|
@REM The 4NT Shell from jp software
|
||||||
|
:4NTArgs
|
||||||
|
set CMD_LINE_ARGS=%$
|
||||||
|
goto WinNTGetScriptDir
|
||||||
|
|
||||||
|
:Win9xArg
|
||||||
|
@REM Slurp the command line arguments. This loop allows for an unlimited number
|
||||||
|
@REM of agruments (up to the command line limit, anyway).
|
||||||
|
set CMD_LINE_ARGS=
|
||||||
|
:Win9xApp
|
||||||
|
if %1a==a goto Win9xGetScriptDir
|
||||||
|
set CMD_LINE_ARGS=%CMD_LINE_ARGS% %1
|
||||||
|
shift
|
||||||
|
goto Win9xApp
|
||||||
|
|
||||||
|
:Win9xGetScriptDir
|
||||||
|
set SAVEDIR=%CD%
|
||||||
|
%0\
|
||||||
|
cd %0\..\..
|
||||||
|
set BASEDIR=%CD%
|
||||||
|
cd %SAVEDIR%
|
||||||
|
set SAVE_DIR=
|
||||||
|
goto repoSetup
|
||||||
|
|
||||||
|
:WinNTGetScriptDir
|
||||||
|
set BASEDIR=%~dp0\..
|
||||||
|
|
||||||
|
:repoSetup
|
||||||
|
|
||||||
|
|
||||||
|
if "%JAVACMD%"=="" set JAVACMD=java
|
||||||
|
|
||||||
|
if "%REPO%"=="" set REPO=%BASEDIR%\lib
|
||||||
|
|
||||||
|
set CLASSPATH="%CLASSPATH%";"%REPO%\*;%BASEDIR%\..\..\dist\solrj-lib\*;%BASEDIR%\..\..\dist\solr-solrj-*;%BASEDIR%\..\..\dist\solr-prometheus-exporter-*"
|
||||||
|
set EXTRA_JVM_ARGUMENTS=-Xmx512m -Dlog4j.configuration=file:%BASEDIR%/conf/log4j.properties
|
||||||
|
goto endInit
|
||||||
|
|
||||||
|
@REM Reaching here means variables are defined and arguments have been captured
|
||||||
|
:endInit
|
||||||
|
|
||||||
|
%JAVACMD% %JAVA_OPTS% %EXTRA_JVM_ARGUMENTS% -classpath %CLASSPATH_PREFIX%;%CLASSPATH% -Dapp.name="solr-exporter" -Dapp.repo="%REPO%" -Dbasedir="%BASEDIR%" com.github.mosuka.solr.prometheus.exporter.SolrExporter %CMD_LINE_ARGS%
|
||||||
|
if ERRORLEVEL 1 goto error
|
||||||
|
goto end
|
||||||
|
|
||||||
|
:error
|
||||||
|
if "%OS%"=="Windows_NT" @endlocal
|
||||||
|
set ERROR_CODE=1
|
||||||
|
|
||||||
|
:end
|
||||||
|
@REM set local scope for the variables with windows NT shell
|
||||||
|
if "%OS%"=="Windows_NT" goto endNT
|
||||||
|
|
||||||
|
@REM For old DOS remove the set variables from ENV - we assume they were not set
|
||||||
|
@REM before we started - at least we don't leave any baggage around
|
||||||
|
set CMD_LINE_ARGS=
|
||||||
|
goto postExec
|
||||||
|
|
||||||
|
:endNT
|
||||||
|
@endlocal
|
||||||
|
|
||||||
|
:postExec
|
||||||
|
|
||||||
|
if "%FORCE_EXIT_ON_ERROR%" == "on" (
|
||||||
|
if %ERROR_CODE% NEQ 0 exit %ERROR_CODE%
|
||||||
|
)
|
||||||
|
|
||||||
|
exit /B %ERROR_CODE%
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
<?xml version="1.0"?>
|
||||||
|
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<project name="solr-prometheus-exporter" default="default">
|
||||||
|
|
||||||
|
<description>
|
||||||
|
Prometheus exporter for exposing metrics from Solr using Metrics API and Search API.
|
||||||
|
</description>
|
||||||
|
|
||||||
|
<import file="../contrib-build.xml"/>
|
||||||
|
|
||||||
|
<target name="compile-core" depends="solr-contrib-build.compile-core"/>
|
||||||
|
<target name="compile-test" depends="common-solr.compile-test"/>
|
||||||
|
</project>
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,22 @@
|
||||||
|
#
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
log4j.rootLogger=INFO, stdout
|
||||||
|
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
||||||
|
log4j.appender.stdout.Target=System.out
|
||||||
|
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||||
|
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss.SSS} %-5p [%c] - %m%n
|
||||||
|
|
@ -0,0 +1,41 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
or more contributor license agreements. See the NOTICE file
|
||||||
|
distributed with this work for additional information
|
||||||
|
regarding copyright ownership. The ASF licenses this file
|
||||||
|
to you under the Apache License, Version 2.0 (the
|
||||||
|
"License"); you may not use this file except in compliance
|
||||||
|
with the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing,
|
||||||
|
software distributed under the License is distributed on an
|
||||||
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
KIND, either express or implied. See the License for the
|
||||||
|
specific language governing permissions and limitations
|
||||||
|
under the License.
|
||||||
|
-->
|
||||||
|
<ivy-module version="2.0">
|
||||||
|
<info organisation="org.apache.solr" module="prometheus"/>
|
||||||
|
<configurations defaultconfmapping="compile->master;test->master">
|
||||||
|
<conf name="compile" transitive="false"/>
|
||||||
|
<conf name="test" transitive="false"/>
|
||||||
|
</configurations>
|
||||||
|
<dependencies>
|
||||||
|
<dependency org="io.prometheus" name="simpleclient" rev="0.0.26" conf="compile"/>
|
||||||
|
<dependency org="io.prometheus" name="simpleclient_common" rev="0.0.26" conf="compile"/>
|
||||||
|
<dependency org="io.prometheus" name="simpleclient_httpserver" rev="0.0.26" conf="compile"/>
|
||||||
|
<dependency org="org.yaml" name="snakeyaml" rev="1.16" conf="compile"/>
|
||||||
|
<dependency org="com.fasterxml.jackson.core" name="jackson-core" rev="2.9.1" conf="compile"/>
|
||||||
|
<dependency org="com.fasterxml.jackson.core" name="jackson-databind" rev="2.9.1" conf="compile"/>
|
||||||
|
<dependency org="com.fasterxml.jackson.core" name="jackson-annotations" rev="2.9.1" conf="compile"/>
|
||||||
|
<dependency org="net.thisptr" name="jackson-jq" rev="0.0.8" conf="compile"/>
|
||||||
|
<dependency org="net.sourceforge.argparse4j" name="argparse4j" rev="0.7.0" conf="compile"/>
|
||||||
|
<dependency org="org.slf4j" name="slf4j-api" rev="1.7.25" conf="compile"/>
|
||||||
|
<dependency org="org.slf4j" name="slf4j-log4j12" rev="1.7.25" conf="compile"/>
|
||||||
|
<dependency org="log4j" name="log4j" rev="1.2.17" conf="compile"/>
|
||||||
|
|
||||||
|
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
|
||||||
|
</dependencies>
|
||||||
|
</ivy-module>
|
||||||
|
|
@ -0,0 +1,402 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.prometheus.collector;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import org.apache.solr.prometheus.collector.config.SolrCollectorConfig;
|
||||||
|
import org.apache.solr.prometheus.scraper.SolrScraper;
|
||||||
|
import org.apache.solr.prometheus.scraper.config.SolrScraperConfig;
|
||||||
|
import io.prometheus.client.Collector;
|
||||||
|
import org.apache.solr.client.solrj.SolrClient;
|
||||||
|
import org.apache.solr.client.solrj.SolrServerException;
|
||||||
|
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
||||||
|
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||||
|
import org.apache.solr.client.solrj.impl.NoOpResponseParser;
|
||||||
|
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||||
|
import org.apache.solr.client.solrj.request.CoreAdminRequest;
|
||||||
|
import org.apache.solr.common.params.CoreAdminParams;
|
||||||
|
import org.apache.solr.common.util.NamedList;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.ExecutionException;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Future;
|
||||||
|
import java.util.concurrent.TimeoutException;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SolrCollector
|
||||||
|
*/
|
||||||
|
public class SolrCollector extends Collector implements Collector.Describable {
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(SolrCollector.class);
|
||||||
|
|
||||||
|
private SolrClient solrClient;
|
||||||
|
private SolrCollectorConfig config = new SolrCollectorConfig();
|
||||||
|
private int numThreads;
|
||||||
|
|
||||||
|
private static ObjectMapper om = new ObjectMapper();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor.
|
||||||
|
*/
|
||||||
|
public SolrCollector(SolrClient solrClient, SolrCollectorConfig config, int numThreads) {
|
||||||
|
this.solrClient = solrClient;
|
||||||
|
this.config = config;
|
||||||
|
this.numThreads = numThreads;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Describe scrape status.
|
||||||
|
*/
|
||||||
|
public List<Collector.MetricFamilySamples> describe() {
|
||||||
|
List<MetricFamilySamples> metricFamilies = new ArrayList<>();
|
||||||
|
metricFamilies.add(new MetricFamilySamples("solr_exporter_duration_seconds", Type.GAUGE, "Time this Solr scrape took, in seconds.", new ArrayList<>()));
|
||||||
|
return metricFamilies;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Collect samples.
|
||||||
|
*/
|
||||||
|
public List<MetricFamilySamples> collect() {
|
||||||
|
// start time of scraping.
|
||||||
|
long startTime = System.nanoTime();
|
||||||
|
|
||||||
|
Map<String, MetricFamilySamples> metricFamilySamplesMap = new LinkedHashMap<>();
|
||||||
|
|
||||||
|
ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
|
||||||
|
List<Future<Map<String, MetricFamilySamples>>> futureList = new ArrayList<>();
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Ping
|
||||||
|
if (config.getPing() != null) {
|
||||||
|
if (solrClient instanceof CloudSolrClient) {
|
||||||
|
List<HttpSolrClient> httpSolrClients = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
httpSolrClients = getHttpSolrClients((CloudSolrClient) solrClient);
|
||||||
|
for (HttpSolrClient httpSolrClient : httpSolrClients) {
|
||||||
|
try {
|
||||||
|
List<String> cores = getCores(httpSolrClient);
|
||||||
|
for (String core : cores) {
|
||||||
|
SolrScraperConfig pingConfig;
|
||||||
|
try {
|
||||||
|
pingConfig = config.getPing().clone();
|
||||||
|
} catch (CloneNotSupportedException e) {
|
||||||
|
logger.error(e.getMessage());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
pingConfig.getQuery().setCore(core);
|
||||||
|
|
||||||
|
SolrScraper scraper = new SolrScraper(httpSolrClient, pingConfig, Arrays.asList("zk_host"), Arrays.asList(((CloudSolrClient) solrClient).getZkHost()));
|
||||||
|
Future<Map<String, MetricFamilySamples>> future = executorService.submit(scraper);
|
||||||
|
futureList.add(future);
|
||||||
|
}
|
||||||
|
} catch (SolrServerException | IOException e) {
|
||||||
|
logger.error(e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// get future
|
||||||
|
for (Future<Map<String, MetricFamilySamples>> future : futureList) {
|
||||||
|
try {
|
||||||
|
Map<String, MetricFamilySamples> m = future.get(60, TimeUnit.SECONDS);
|
||||||
|
mergeMetrics(metricFamilySamplesMap, m);
|
||||||
|
} catch (InterruptedException | ExecutionException | TimeoutException e) {
|
||||||
|
logger.error(e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (SolrServerException | IOException e) {
|
||||||
|
logger.error(e.getMessage());
|
||||||
|
} finally {
|
||||||
|
for (HttpSolrClient httpSolrClient : httpSolrClients) {
|
||||||
|
try {
|
||||||
|
httpSolrClient.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
logger.error(e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
List<String> collections = getCollections((CloudSolrClient) solrClient);
|
||||||
|
for (String collection : collections) {
|
||||||
|
SolrScraperConfig pingConfig;
|
||||||
|
try {
|
||||||
|
pingConfig = config.getPing().clone();
|
||||||
|
} catch (CloneNotSupportedException e) {
|
||||||
|
logger.error(e.getMessage());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
pingConfig.getQuery().setCollection(collection);
|
||||||
|
LinkedHashMap<String, String> distrib = new LinkedHashMap<>();
|
||||||
|
distrib.put("distrib", "true");
|
||||||
|
pingConfig.getQuery().setParams(Collections.singletonList(distrib));
|
||||||
|
|
||||||
|
SolrScraper scraper = new SolrScraper(solrClient, pingConfig);
|
||||||
|
Future<Map<String, MetricFamilySamples>> future = executorService.submit(scraper);
|
||||||
|
futureList.add(future);
|
||||||
|
}
|
||||||
|
} catch (SolrServerException | IOException e) {
|
||||||
|
logger.error(e.getMessage());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
List<String> cores = getCores((HttpSolrClient) solrClient);
|
||||||
|
for (String core : cores) {
|
||||||
|
SolrScraperConfig pingConfig = new SolrScraperConfig();
|
||||||
|
pingConfig.setQuery(config.getPing().getQuery());
|
||||||
|
pingConfig.getQuery().setCore(core);
|
||||||
|
|
||||||
|
pingConfig.setJsonQueries(config.getPing().getJsonQueries());
|
||||||
|
|
||||||
|
SolrScraper scraper = new SolrScraper(solrClient, pingConfig);
|
||||||
|
Future<Map<String, MetricFamilySamples>> future = executorService.submit(scraper);
|
||||||
|
futureList.add(future);
|
||||||
|
}
|
||||||
|
} catch (SolrServerException | IOException e) {
|
||||||
|
logger.error(e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Metrics
|
||||||
|
if (config.getMetrics() != null) {
|
||||||
|
if (solrClient instanceof CloudSolrClient) {
|
||||||
|
List<HttpSolrClient> httpSolrClients = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
httpSolrClients = getHttpSolrClients((CloudSolrClient) solrClient);
|
||||||
|
for (HttpSolrClient httpSolrClient : httpSolrClients) {
|
||||||
|
SolrScraper scraper = new SolrScraper(httpSolrClient, config.getMetrics(), Arrays.asList("zk_host"), Arrays.asList(((CloudSolrClient) solrClient).getZkHost()));
|
||||||
|
Future<Map<String, MetricFamilySamples>> future = executorService.submit(scraper);
|
||||||
|
futureList.add(future);
|
||||||
|
}
|
||||||
|
|
||||||
|
// get future
|
||||||
|
for (Future<Map<String, MetricFamilySamples>> future : futureList) {
|
||||||
|
try {
|
||||||
|
Map<String, MetricFamilySamples> m = future.get(60, TimeUnit.SECONDS);
|
||||||
|
mergeMetrics(metricFamilySamplesMap, m);
|
||||||
|
} catch (InterruptedException | ExecutionException | TimeoutException e) {
|
||||||
|
logger.error(e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (SolrServerException | IOException e) {
|
||||||
|
logger.error(e.getMessage());
|
||||||
|
} finally {
|
||||||
|
for (HttpSolrClient httpSolrClient : httpSolrClients) {
|
||||||
|
try {
|
||||||
|
httpSolrClient.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
logger.error(e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
SolrScraper scraper = new SolrScraper(solrClient, config.getMetrics());
|
||||||
|
Future<Map<String, MetricFamilySamples>> future = executorService.submit(scraper);
|
||||||
|
futureList.add(future);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collections
|
||||||
|
if (config.getCollections() != null) {
|
||||||
|
if (solrClient instanceof CloudSolrClient) {
|
||||||
|
SolrScraper scraper = new SolrScraper(solrClient, config.getCollections());
|
||||||
|
Future<Map<String, MetricFamilySamples>> future = executorService.submit(scraper);
|
||||||
|
futureList.add(future);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Query
|
||||||
|
if (config.getQueries() != null) {
|
||||||
|
for (SolrScraperConfig c : config.getQueries()) {
|
||||||
|
SolrScraper scraper = new SolrScraper(solrClient, c);
|
||||||
|
Future<Map<String, MetricFamilySamples>> future = executorService.submit(scraper);
|
||||||
|
futureList.add(future);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// get future
|
||||||
|
for (Future<Map<String, MetricFamilySamples>> future : futureList) {
|
||||||
|
try {
|
||||||
|
Map<String, MetricFamilySamples> m = future.get(60, TimeUnit.SECONDS);
|
||||||
|
mergeMetrics(metricFamilySamplesMap, m);
|
||||||
|
} catch (InterruptedException | ExecutionException | TimeoutException e) {
|
||||||
|
logger.error(e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
executorService.shutdown();
|
||||||
|
}
|
||||||
|
|
||||||
|
// return value
|
||||||
|
List<MetricFamilySamples> metricFamiliesSamplesList = new ArrayList<>();
|
||||||
|
|
||||||
|
// add solr metrics
|
||||||
|
for (String gaugeMetricName : metricFamilySamplesMap.keySet()) {
|
||||||
|
MetricFamilySamples metricFamilySamples = metricFamilySamplesMap.get(gaugeMetricName);
|
||||||
|
if (metricFamilySamples.samples.size() > 0) {
|
||||||
|
metricFamiliesSamplesList.add(metricFamilySamples);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// add scrape duration metric
|
||||||
|
List<MetricFamilySamples.Sample> durationSample = new ArrayList<>();
|
||||||
|
durationSample.add(new MetricFamilySamples.Sample("solr_exporter_duration_seconds", new ArrayList<>(), new ArrayList<>(), (System.nanoTime() - startTime) / 1.0E9));
|
||||||
|
metricFamiliesSamplesList.add(new MetricFamilySamples("solr_exporter_duration_seconds", Type.GAUGE, "Time this Solr exporter took, in seconds.", durationSample));
|
||||||
|
|
||||||
|
return metricFamiliesSamplesList;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Merge metrics.
|
||||||
|
*/
|
||||||
|
private Map<String, MetricFamilySamples> mergeMetrics(Map<String, MetricFamilySamples> metrics1, Map<String, MetricFamilySamples> metrics2) {
|
||||||
|
// marge MetricFamilySamples
|
||||||
|
for (String k : metrics2.keySet()) {
|
||||||
|
if (metrics1.containsKey(k)) {
|
||||||
|
for (MetricFamilySamples.Sample sample : metrics2.get(k).samples) {
|
||||||
|
if (!metrics1.get(k).samples.contains(sample)) {
|
||||||
|
metrics1.get(k).samples.add(sample);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
metrics1.put(k, metrics2.get(k));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return metrics1;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get target cores via CoreAdminAPI.
|
||||||
|
*/
|
||||||
|
public static List<String> getCores(HttpSolrClient httpSolrClient) throws SolrServerException, IOException {
|
||||||
|
List<String> cores = new ArrayList<>();
|
||||||
|
|
||||||
|
NoOpResponseParser responseParser = new NoOpResponseParser();
|
||||||
|
responseParser.setWriterType("json");
|
||||||
|
|
||||||
|
httpSolrClient.setParser(responseParser);
|
||||||
|
|
||||||
|
CoreAdminRequest coreAdminRequest = new CoreAdminRequest();
|
||||||
|
coreAdminRequest.setAction(CoreAdminParams.CoreAdminAction.STATUS);
|
||||||
|
coreAdminRequest.setIndexInfoNeeded(false);
|
||||||
|
|
||||||
|
NamedList<Object> coreAdminResponse = httpSolrClient.request(coreAdminRequest);
|
||||||
|
|
||||||
|
JsonNode statusJsonNode = om.readTree((String) coreAdminResponse.get("response")).get("status");
|
||||||
|
|
||||||
|
for (Iterator<JsonNode> i = statusJsonNode.iterator(); i.hasNext(); ) {
|
||||||
|
String core = i.next().get("name").textValue();
|
||||||
|
if (!cores.contains(core)) {
|
||||||
|
cores.add(core);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return cores;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get target cores via CollectionsAPI.
|
||||||
|
*/
|
||||||
|
public static List<String> getCollections(CloudSolrClient cloudSolrClient) throws SolrServerException, IOException {
|
||||||
|
List<String> collections = new ArrayList<>();
|
||||||
|
|
||||||
|
NoOpResponseParser responseParser = new NoOpResponseParser();
|
||||||
|
responseParser.setWriterType("json");
|
||||||
|
|
||||||
|
cloudSolrClient.setParser(responseParser);
|
||||||
|
|
||||||
|
CollectionAdminRequest collectionAdminRequest = new CollectionAdminRequest.List();
|
||||||
|
|
||||||
|
NamedList<Object> collectionAdminResponse = cloudSolrClient.request(collectionAdminRequest);
|
||||||
|
|
||||||
|
JsonNode collectionsJsonNode = om.readTree((String) collectionAdminResponse.get("response")).get("collections");
|
||||||
|
|
||||||
|
for (Iterator<JsonNode> i = collectionsJsonNode.iterator(); i.hasNext(); ) {
|
||||||
|
String collection = i.next().textValue();
|
||||||
|
if (!collections.contains(collection)) {
|
||||||
|
collections.add(collection);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return collections;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get base urls via CollectionsAPI.
|
||||||
|
*/
|
||||||
|
private List<String> getBaseUrls(CloudSolrClient cloudSolrClient) throws SolrServerException, IOException {
|
||||||
|
List<String> baseUrls = new ArrayList<>();
|
||||||
|
|
||||||
|
NoOpResponseParser responseParser = new NoOpResponseParser();
|
||||||
|
responseParser.setWriterType("json");
|
||||||
|
|
||||||
|
cloudSolrClient.setParser(responseParser);
|
||||||
|
|
||||||
|
CollectionAdminRequest collectionAdminRequest = new CollectionAdminRequest.ClusterStatus();
|
||||||
|
|
||||||
|
NamedList<Object> collectionAdminResponse = cloudSolrClient.request(collectionAdminRequest);
|
||||||
|
|
||||||
|
List<JsonNode> baseUrlJsonNode = om.readTree((String) collectionAdminResponse.get("response")).findValues("base_url");
|
||||||
|
|
||||||
|
for (Iterator<JsonNode> i = baseUrlJsonNode.iterator(); i.hasNext(); ) {
|
||||||
|
String baseUrl = i.next().textValue();
|
||||||
|
if (!baseUrls.contains(baseUrl)) {
|
||||||
|
baseUrls.add(baseUrl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return baseUrls;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get HTTP Solr Clients
|
||||||
|
*/
|
||||||
|
private List<HttpSolrClient> getHttpSolrClients(CloudSolrClient cloudSolrClient) throws SolrServerException, IOException {
|
||||||
|
List<HttpSolrClient> solrClients = new ArrayList<>();
|
||||||
|
|
||||||
|
for (String baseUrl : getBaseUrls(cloudSolrClient)) {
|
||||||
|
NoOpResponseParser responseParser = new NoOpResponseParser();
|
||||||
|
responseParser.setWriterType("json");
|
||||||
|
|
||||||
|
HttpSolrClient.Builder builder = new HttpSolrClient.Builder();
|
||||||
|
builder.withBaseSolrUrl(baseUrl);
|
||||||
|
|
||||||
|
HttpSolrClient httpSolrClient = builder.build();
|
||||||
|
httpSolrClient.setParser(responseParser);
|
||||||
|
|
||||||
|
solrClients.add(httpSolrClient);
|
||||||
|
}
|
||||||
|
|
||||||
|
return solrClients;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.prometheus.collector.config;
|
||||||
|
|
||||||
|
import org.apache.solr.prometheus.scraper.config.SolrScraperConfig;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SolrCollectorConfig
|
||||||
|
*/
|
||||||
|
public class SolrCollectorConfig {
|
||||||
|
private SolrScraperConfig ping = new SolrScraperConfig();
|
||||||
|
private SolrScraperConfig metrics = new SolrScraperConfig();
|
||||||
|
private SolrScraperConfig collections = new SolrScraperConfig();
|
||||||
|
private List<SolrScraperConfig> queries = new ArrayList<>();
|
||||||
|
|
||||||
|
public SolrScraperConfig getPing() {
|
||||||
|
return ping;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPing(SolrScraperConfig ping) {
|
||||||
|
this.ping = ping;
|
||||||
|
}
|
||||||
|
|
||||||
|
public SolrScraperConfig getMetrics() {
|
||||||
|
return metrics;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setMetrics(SolrScraperConfig metrics) {
|
||||||
|
this.metrics = metrics;
|
||||||
|
}
|
||||||
|
|
||||||
|
public SolrScraperConfig getCollections() {
|
||||||
|
return collections;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCollections(SolrScraperConfig collections) {
|
||||||
|
this.collections = collections;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<SolrScraperConfig> getQueries() {
|
||||||
|
return queries;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setQueries(List<SolrScraperConfig> queries) {
|
||||||
|
this.queries = queries;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,254 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.prometheus.exporter;
|
||||||
|
|
||||||
|
import org.apache.solr.prometheus.collector.SolrCollector;
|
||||||
|
import org.apache.solr.prometheus.collector.config.SolrCollectorConfig;
|
||||||
|
import io.prometheus.client.CollectorRegistry;
|
||||||
|
import io.prometheus.client.Counter;
|
||||||
|
import io.prometheus.client.exporter.HTTPServer;
|
||||||
|
import net.sourceforge.argparse4j.ArgumentParsers;
|
||||||
|
import net.sourceforge.argparse4j.impl.Arguments;
|
||||||
|
import net.sourceforge.argparse4j.inf.ArgumentParser;
|
||||||
|
import net.sourceforge.argparse4j.inf.ArgumentParserException;
|
||||||
|
import net.sourceforge.argparse4j.inf.Namespace;
|
||||||
|
import org.apache.solr.client.solrj.SolrClient;
|
||||||
|
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
||||||
|
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||||
|
import org.apache.solr.client.solrj.impl.NoOpResponseParser;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.yaml.snakeyaml.Yaml;
|
||||||
|
|
||||||
|
import javax.management.MalformedObjectNameException;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileReader;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Properties;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SolrExporter
|
||||||
|
*/
|
||||||
|
public class SolrExporter {
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(SolrExporter.class);
|
||||||
|
|
||||||
|
private static final String[] ARG_PORT_FLAGS = { "-p", "--port" };
|
||||||
|
private static final String ARG_PORT_METAVAR = "PORT";
|
||||||
|
private static final String ARG_PORT_DEST = "port";
|
||||||
|
private static final Integer ARG_PORT_DEFAULT = 9983;
|
||||||
|
private static final String ARG_PORT_HELP = "solr-exporter listen port";
|
||||||
|
|
||||||
|
private static final String[] ARG_BASE_URL_FLAGS = { "-b", "--baseurl" };
|
||||||
|
private static final String ARG_BASE_URL_METAVAR = "BASE_URL";
|
||||||
|
private static final String ARG_BASE_URL_DEST = "baseUrl";
|
||||||
|
private static final String ARG_BASE_URL_DEFAULT = "";
|
||||||
|
private static final String ARG_BASE_URL_HELP = "specify Solr base URL when connecting to Solr in standalone mode (for example 'http://localhost:8983/solr')";
|
||||||
|
|
||||||
|
private static final String[] ARG_ZK_HOST_FLAGS = { "-z", "--zkhost" };
|
||||||
|
private static final String ARG_ZK_HOST_METAVAR = "ZK_HOST";
|
||||||
|
private static final String ARG_ZK_HOST_DEST = "zkHost";
|
||||||
|
private static final String ARG_ZK_HOST_DEFAULT = "";
|
||||||
|
private static final String ARG_ZK_HOST_HELP = "specify ZooKeeper connection string when connecting to Solr in SolrCloud mode (for example 'localhost:2181/solr')";
|
||||||
|
|
||||||
|
private static final String[] ARG_CONFIG_FLAGS = { "-f", "--config-file" };
|
||||||
|
private static final String ARG_CONFIG_METAVAR = "CONFIG";
|
||||||
|
private static final String ARG_CONFIG_DEST = "configFile";
|
||||||
|
private static final String ARG_CONFIG_DEFAULT = "./conf/config.yml";
|
||||||
|
private static final String ARG_CONFIG_HELP = "specify configuration file";
|
||||||
|
|
||||||
|
private static final String[] ARG_NUM_THREADS_FLAGS = { "-n", "--num-thread" };
|
||||||
|
private static final String ARG_NUM_THREADS_METAVAR = "NUM_THREADS";
|
||||||
|
private static final String ARG_NUM_THREADS_DEST = "numThreads";
|
||||||
|
private static final Integer ARG_NUM_THREADS_DEFAULT = 1;
|
||||||
|
private static final String ARG_NUM_THREADS_HELP = "specify number of threads";
|
||||||
|
|
||||||
|
private int port;
|
||||||
|
private SolrClient solrClient;
|
||||||
|
private SolrCollectorConfig config;
|
||||||
|
private int numThreads;
|
||||||
|
|
||||||
|
CollectorRegistry registry = new CollectorRegistry();
|
||||||
|
|
||||||
|
private HTTPServer httpServer;
|
||||||
|
private SolrCollector collector;
|
||||||
|
|
||||||
|
public static final Counter scrapeErrorTotal = Counter.build()
|
||||||
|
.name("solr_exporter_scrape_error_total")
|
||||||
|
.help("Number of scrape error.").register();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor.
|
||||||
|
*/
|
||||||
|
public SolrExporter(int port, SolrClient solrClient, File configFile, int numThreads) throws IOException {
|
||||||
|
this(port, solrClient, new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class), numThreads);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor.
|
||||||
|
*/
|
||||||
|
public SolrExporter(int port, SolrClient solrClient, SolrCollectorConfig config, int numThreads) {
|
||||||
|
super();
|
||||||
|
|
||||||
|
this.port = port;
|
||||||
|
this.solrClient = solrClient;
|
||||||
|
this.config = config;
|
||||||
|
this.numThreads = numThreads;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start HTTP server for exporting Solr metrics.
|
||||||
|
*/
|
||||||
|
public void start() throws MalformedObjectNameException, IOException {
|
||||||
|
InetSocketAddress socket = new InetSocketAddress(port);
|
||||||
|
|
||||||
|
this.collector = new SolrCollector(solrClient, config, numThreads);
|
||||||
|
|
||||||
|
this.registry.register(this.collector);
|
||||||
|
this.registry.register(scrapeErrorTotal);
|
||||||
|
|
||||||
|
this.httpServer = new HTTPServer(socket, this.registry);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stop HTTP server for exporting Solr metrics.
|
||||||
|
*/
|
||||||
|
public void stop() throws IOException {
|
||||||
|
this.httpServer.stop();
|
||||||
|
this.registry.unregister(this.collector);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create Solr client
|
||||||
|
*/
|
||||||
|
private static SolrClient createClient(String connStr) {
|
||||||
|
SolrClient solrClient;
|
||||||
|
|
||||||
|
Pattern baseUrlPattern = Pattern.compile("^https?:\\/\\/[\\w\\/:%#\\$&\\?\\(\\)~\\.=\\+\\-]+$");
|
||||||
|
Pattern zkHostPattern = Pattern.compile("^(?<host>[^\\/]+)(?<chroot>|(?:\\/.*))$");
|
||||||
|
Matcher matcher;
|
||||||
|
|
||||||
|
matcher = baseUrlPattern.matcher(connStr);
|
||||||
|
if (matcher.matches()) {
|
||||||
|
NoOpResponseParser responseParser = new NoOpResponseParser();
|
||||||
|
responseParser.setWriterType("json");
|
||||||
|
|
||||||
|
HttpSolrClient.Builder builder = new HttpSolrClient.Builder();
|
||||||
|
builder.withBaseSolrUrl(connStr);
|
||||||
|
|
||||||
|
HttpSolrClient httpSolrClient = builder.build();
|
||||||
|
httpSolrClient.setParser(responseParser);
|
||||||
|
|
||||||
|
solrClient = httpSolrClient;
|
||||||
|
} else {
|
||||||
|
String host = "";
|
||||||
|
String chroot = "";
|
||||||
|
|
||||||
|
matcher = zkHostPattern.matcher(connStr);
|
||||||
|
if (matcher.matches()) {
|
||||||
|
host = matcher.group("host") != null ? matcher.group("host") : "";
|
||||||
|
chroot = matcher.group("chroot") != null ? matcher.group("chroot") : "";
|
||||||
|
}
|
||||||
|
|
||||||
|
NoOpResponseParser responseParser = new NoOpResponseParser();
|
||||||
|
responseParser.setWriterType("json");
|
||||||
|
|
||||||
|
CloudSolrClient.Builder builder = new CloudSolrClient.Builder();
|
||||||
|
if (host.contains(",")) {
|
||||||
|
List<String> hosts = new ArrayList<>();
|
||||||
|
for (String h : host.split(",")) {
|
||||||
|
if (h != null && !h.equals("")) {
|
||||||
|
hosts.add(h.trim());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
builder.withZkHost(hosts);
|
||||||
|
} else {
|
||||||
|
builder.withZkHost(host);
|
||||||
|
}
|
||||||
|
if (chroot.equals("")) {
|
||||||
|
builder.withZkChroot("/");
|
||||||
|
} else {
|
||||||
|
builder.withZkChroot(chroot);
|
||||||
|
}
|
||||||
|
|
||||||
|
CloudSolrClient cloudSolrClient = builder.build();
|
||||||
|
cloudSolrClient.setParser(responseParser);
|
||||||
|
|
||||||
|
solrClient = cloudSolrClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
return solrClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Entry point of SolrExporter.
|
||||||
|
*/
|
||||||
|
public static void main( String[] args ) {
|
||||||
|
ArgumentParser parser = ArgumentParsers.newArgumentParser(SolrCollector.class.getSimpleName())
|
||||||
|
.description("Prometheus exporter for Apache Solr.");
|
||||||
|
|
||||||
|
parser.addArgument(ARG_PORT_FLAGS)
|
||||||
|
.metavar(ARG_PORT_METAVAR).dest(ARG_PORT_DEST).type(Integer.class)
|
||||||
|
.setDefault(ARG_PORT_DEFAULT).help(ARG_PORT_HELP);
|
||||||
|
|
||||||
|
parser.addArgument(ARG_BASE_URL_FLAGS)
|
||||||
|
.metavar(ARG_BASE_URL_METAVAR).dest(ARG_BASE_URL_DEST).type(String.class)
|
||||||
|
.setDefault(ARG_BASE_URL_DEFAULT).help(ARG_BASE_URL_HELP);
|
||||||
|
|
||||||
|
parser.addArgument(ARG_ZK_HOST_FLAGS)
|
||||||
|
.metavar(ARG_ZK_HOST_METAVAR).dest(ARG_ZK_HOST_DEST).type(String.class)
|
||||||
|
.setDefault(ARG_ZK_HOST_DEFAULT).help(ARG_ZK_HOST_HELP);
|
||||||
|
|
||||||
|
parser.addArgument(ARG_CONFIG_FLAGS)
|
||||||
|
.metavar(ARG_CONFIG_METAVAR).dest(ARG_CONFIG_DEST).type(String.class)
|
||||||
|
.setDefault(ARG_CONFIG_DEFAULT).help(ARG_CONFIG_HELP);
|
||||||
|
|
||||||
|
parser.addArgument(ARG_NUM_THREADS_FLAGS)
|
||||||
|
.metavar(ARG_NUM_THREADS_METAVAR).dest(ARG_NUM_THREADS_DEST).type(Integer.class)
|
||||||
|
.setDefault(ARG_NUM_THREADS_DEFAULT).help(ARG_NUM_THREADS_HELP);
|
||||||
|
|
||||||
|
try {
|
||||||
|
Namespace res = parser.parseArgs(args);
|
||||||
|
|
||||||
|
int port = res.getInt(ARG_PORT_DEST);
|
||||||
|
|
||||||
|
String connStr = "http://localhost:8983/solr";
|
||||||
|
if (!res.getString(ARG_BASE_URL_DEST).equals("")) {
|
||||||
|
connStr = res.getString(ARG_BASE_URL_DEST);
|
||||||
|
} else if (!res.getString(ARG_ZK_HOST_DEST).equals("")) {
|
||||||
|
connStr = res.getString(ARG_ZK_HOST_DEST);
|
||||||
|
}
|
||||||
|
|
||||||
|
File configFile = new File(res.getString(ARG_CONFIG_DEST));
|
||||||
|
int numThreads = res.getInt(ARG_NUM_THREADS_DEST);
|
||||||
|
|
||||||
|
SolrClient solrClient = createClient(connStr);
|
||||||
|
|
||||||
|
SolrExporter solrExporter = new SolrExporter(port, solrClient, configFile, numThreads);
|
||||||
|
solrExporter.start();
|
||||||
|
logger.info("Start server");
|
||||||
|
} catch (MalformedObjectNameException | IOException e) {
|
||||||
|
logger.error("Start server failed: " + e.toString());
|
||||||
|
} catch (ArgumentParserException e) {
|
||||||
|
parser.handleError(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,218 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.prometheus.scraper;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import org.apache.solr.prometheus.exporter.SolrExporter;
|
||||||
|
import org.apache.solr.prometheus.scraper.config.SolrQueryConfig;
|
||||||
|
import org.apache.solr.prometheus.scraper.config.SolrScraperConfig;
|
||||||
|
import io.prometheus.client.Collector;
|
||||||
|
import net.thisptr.jackson.jq.JsonQuery;
|
||||||
|
import net.thisptr.jackson.jq.exception.JsonQueryException;
|
||||||
|
import org.apache.solr.client.solrj.SolrClient;
|
||||||
|
import org.apache.solr.client.solrj.SolrServerException;
|
||||||
|
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
||||||
|
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||||
|
import org.apache.solr.client.solrj.request.QueryRequest;
|
||||||
|
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||||
|
import org.apache.solr.common.util.NamedList;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.Callable;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SolrScraper
|
||||||
|
*/
|
||||||
|
public class SolrScraper implements Callable<Map<String, Collector.MetricFamilySamples>> {
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(SolrScraper.class);
|
||||||
|
|
||||||
|
private SolrClient solrClient;
|
||||||
|
private SolrScraperConfig scraperConfig;
|
||||||
|
|
||||||
|
private List<String> labelNames;
|
||||||
|
private List<String> labelValues;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor.
|
||||||
|
*/
|
||||||
|
public SolrScraper(SolrClient solrClient, SolrScraperConfig scraperConfig) {
|
||||||
|
this(solrClient, scraperConfig, new ArrayList<>(), new ArrayList<>());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor.
|
||||||
|
*/
|
||||||
|
public SolrScraper(SolrClient solrClient, SolrScraperConfig scraperConfig, List<String> labelNames, List<String> labelValues) {
|
||||||
|
super();
|
||||||
|
|
||||||
|
this.solrClient = solrClient;
|
||||||
|
this.scraperConfig = scraperConfig;
|
||||||
|
|
||||||
|
this.labelNames = labelNames;
|
||||||
|
this.labelValues = labelValues;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute collectResponse
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public Map<String, Collector.MetricFamilySamples> call() throws Exception {
|
||||||
|
return collectResponse(this.solrClient, this.scraperConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Collect facet count.
|
||||||
|
*/
|
||||||
|
public Map<String, Collector.MetricFamilySamples> collectResponse(SolrClient solrClient, SolrScraperConfig scraperConfig) {
|
||||||
|
Map<String, Collector.MetricFamilySamples> metricFamilySamplesMap = new LinkedHashMap<>();
|
||||||
|
|
||||||
|
try {
|
||||||
|
SolrQueryConfig queryConfig = scraperConfig.getQuery();
|
||||||
|
|
||||||
|
// create Solr request parameters
|
||||||
|
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||||
|
for (Map<String, String> param : queryConfig.getParams()) {
|
||||||
|
for (String name : param.keySet()) {
|
||||||
|
Object obj = param.get(name);
|
||||||
|
if (obj instanceof Number) {
|
||||||
|
params.add(name, obj.toString());
|
||||||
|
} else {
|
||||||
|
params.add(name, param.get(name));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// create Solr queryConfig request
|
||||||
|
QueryRequest queryRequest = new QueryRequest(params);
|
||||||
|
queryRequest.setPath(queryConfig.getPath());
|
||||||
|
|
||||||
|
// invoke Solr
|
||||||
|
NamedList<Object> queryResponse = null;
|
||||||
|
if (queryConfig.getCore().equals("") && queryConfig.getCollection().equals("")) {
|
||||||
|
queryResponse = solrClient.request(queryRequest);
|
||||||
|
} else if (!queryConfig.getCore().equals("")) {
|
||||||
|
queryResponse = solrClient.request(queryRequest, queryConfig.getCore());
|
||||||
|
} else if (!queryConfig.getCollection().equals("")) {
|
||||||
|
queryResponse = solrClient.request(queryRequest, queryConfig.getCollection());
|
||||||
|
}
|
||||||
|
|
||||||
|
ObjectMapper om = new ObjectMapper();
|
||||||
|
|
||||||
|
JsonNode metricsJson = om.readTree((String) queryResponse.get("response"));
|
||||||
|
|
||||||
|
List<JsonQuery> jqs = new ArrayList<>();
|
||||||
|
for (String jsonQuery : scraperConfig.getJsonQueries()) {
|
||||||
|
JsonQuery compiledJsonQuery = JsonQuery.compile(jsonQuery);
|
||||||
|
jqs.add(compiledJsonQuery);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < jqs.size(); i++) {
|
||||||
|
JsonQuery q = jqs.get(i);
|
||||||
|
try {
|
||||||
|
List<JsonNode> results = q.apply(metricsJson);
|
||||||
|
for (JsonNode result : results) {
|
||||||
|
String type = result.get("type").textValue();
|
||||||
|
String name = result.get("name").textValue();
|
||||||
|
String help = result.get("help").textValue();
|
||||||
|
Double value = result.get("value").doubleValue();
|
||||||
|
ArrayList<String> labelNames = new ArrayList<>(this.labelNames);
|
||||||
|
ArrayList<String> labelValues = new ArrayList<>(this.labelValues);
|
||||||
|
|
||||||
|
if (solrClient instanceof CloudSolrClient) {
|
||||||
|
labelNames.add("zk_host");
|
||||||
|
labelValues.add(((CloudSolrClient) solrClient).getZkHost());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!scraperConfig.getQuery().getCollection().equals("")) {
|
||||||
|
labelNames.add("collection");
|
||||||
|
labelValues.add(scraperConfig.getQuery().getCollection());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (solrClient instanceof HttpSolrClient) {
|
||||||
|
labelNames.add("base_url");
|
||||||
|
labelValues.add(((HttpSolrClient) solrClient).getBaseURL());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!scraperConfig.getQuery().getCore().equals("")) {
|
||||||
|
labelNames.add("core");
|
||||||
|
labelValues.add(scraperConfig.getQuery().getCore());
|
||||||
|
}
|
||||||
|
|
||||||
|
for(Iterator<JsonNode> ite = result.get("label_names").iterator();ite.hasNext();){
|
||||||
|
JsonNode item = ite.next();
|
||||||
|
labelNames.add(item.textValue());
|
||||||
|
}
|
||||||
|
for(Iterator<JsonNode> ite = result.get("label_values").iterator();ite.hasNext();){
|
||||||
|
JsonNode item = ite.next();
|
||||||
|
labelValues.add(item.textValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (labelNames.indexOf("core") < 0 && labelNames.indexOf("collection") >= 0 && labelNames.indexOf("shard") >= 0 && labelNames.indexOf("replica") >= 0) {
|
||||||
|
if (labelValues.get(labelNames.indexOf("collection")).equals("-") && labelValues.get(labelNames.indexOf("shard")).equals("-") && labelValues.get(labelNames.indexOf("replica")).equals("-")) {
|
||||||
|
labelNames.add("core");
|
||||||
|
labelValues.add("-");
|
||||||
|
} else {
|
||||||
|
StringBuffer sb = new StringBuffer();
|
||||||
|
sb.append(labelValues.get(labelNames.indexOf("collection")))
|
||||||
|
.append("_")
|
||||||
|
.append(labelValues.get(labelNames.indexOf("shard")))
|
||||||
|
.append("_")
|
||||||
|
.append(labelValues.get(labelNames.indexOf("replica")));
|
||||||
|
|
||||||
|
labelNames.add("core");
|
||||||
|
labelValues.add(sb.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!metricFamilySamplesMap.containsKey(name)) {
|
||||||
|
Collector.MetricFamilySamples metricFamilySamples = new Collector.MetricFamilySamples(
|
||||||
|
name,
|
||||||
|
Collector.Type.valueOf(type),
|
||||||
|
help,
|
||||||
|
new ArrayList<>()
|
||||||
|
);
|
||||||
|
metricFamilySamplesMap.put(name, metricFamilySamples);
|
||||||
|
}
|
||||||
|
|
||||||
|
Collector.MetricFamilySamples.Sample sample = new Collector.MetricFamilySamples.Sample(name, labelNames, labelValues, value);
|
||||||
|
|
||||||
|
if (!metricFamilySamplesMap.get(name).samples.contains(sample)) {
|
||||||
|
metricFamilySamplesMap.get(name).samples.add(sample);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (JsonQueryException e) {
|
||||||
|
logger.error(e.toString() + " " + q.toString());
|
||||||
|
SolrExporter.scrapeErrorTotal.inc();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (HttpSolrClient.RemoteSolrException | SolrServerException | IOException e) {
|
||||||
|
logger.error(e.toString());
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error(e.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
return metricFamilySamplesMap;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,99 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.prometheus.scraper.config;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SolrQueryConfig
|
||||||
|
*/
|
||||||
|
public class SolrQueryConfig implements Cloneable {
|
||||||
|
private String core = "";
|
||||||
|
private String collection = "";
|
||||||
|
private String path = "";
|
||||||
|
private List<LinkedHashMap<String, String>> params = new ArrayList<>();
|
||||||
|
|
||||||
|
public String getCore() {
|
||||||
|
return core;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCore(String core) {
|
||||||
|
this.core = core;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCollection() {
|
||||||
|
return collection;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCollection(String collection) {
|
||||||
|
this.collection = collection;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPath() {
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPath(String path) {
|
||||||
|
this.path = path;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<LinkedHashMap<String, String>> getParams() {
|
||||||
|
return params;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setParams(List<LinkedHashMap<String, String>> params) {
|
||||||
|
this.params = params;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getParamsString() {
|
||||||
|
StringBuffer buffer = new StringBuffer();
|
||||||
|
|
||||||
|
for(Iterator<LinkedHashMap<String, String>> i = getParams().iterator(); i.hasNext(); ) {
|
||||||
|
LinkedHashMap<String, String> param = i.next();
|
||||||
|
for(Iterator<String> j = param.keySet().iterator(); j.hasNext(); ) {
|
||||||
|
String name = j.next();
|
||||||
|
buffer.append(name).append("=").append(param.get(name));
|
||||||
|
if (j.hasNext()) {
|
||||||
|
buffer.append("&");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (i.hasNext()) {
|
||||||
|
buffer.append("&");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return buffer.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public SolrQueryConfig clone() throws CloneNotSupportedException {
|
||||||
|
SolrQueryConfig queryConfig = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
queryConfig = (SolrQueryConfig) super.clone();
|
||||||
|
queryConfig.setCore(new String(this.core));
|
||||||
|
queryConfig.setCollection(new String(this.collection));
|
||||||
|
queryConfig.setParams(new ArrayList<>(this.params));
|
||||||
|
}catch (Exception e){
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
|
return queryConfig;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,60 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.prometheus.scraper.config;
|
||||||
|
|
||||||
|
import net.thisptr.jackson.jq.exception.JsonQueryException;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SolrScraperConfig
|
||||||
|
*/
|
||||||
|
public class SolrScraperConfig implements Cloneable {
|
||||||
|
private SolrQueryConfig query = new SolrQueryConfig();
|
||||||
|
private List<String> jsonQueries = new ArrayList<>();
|
||||||
|
|
||||||
|
public SolrQueryConfig getQuery() {
|
||||||
|
return this.query;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setQuery(SolrQueryConfig query) {
|
||||||
|
this.query = query;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<String> getJsonQueries() {
|
||||||
|
return jsonQueries;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setJsonQueries(List<String> jsonQueries) throws JsonQueryException {
|
||||||
|
this.jsonQueries = jsonQueries;
|
||||||
|
}
|
||||||
|
|
||||||
|
public SolrScraperConfig clone() throws CloneNotSupportedException {
|
||||||
|
SolrScraperConfig scraperConfig = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
scraperConfig = (SolrScraperConfig) super.clone();
|
||||||
|
scraperConfig.setQuery(this.query.clone());
|
||||||
|
scraperConfig.setJsonQueries(new ArrayList<>(this.jsonQueries));
|
||||||
|
}catch (Exception e){
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
|
return scraperConfig;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
Apache Solr Search Server: Solr Prometheus Exporter contrib
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,22 @@
|
||||||
|
#
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
log4j.rootLogger=INFO, stdout
|
||||||
|
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
||||||
|
log4j.appender.stdout.Target=System.out
|
||||||
|
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||||
|
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss.SSS} %-5p [%c] - %m%n
|
||||||
|
|
@ -0,0 +1,412 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8" ?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!--
|
||||||
|
|
||||||
|
This example schema is the recommended starting point for users.
|
||||||
|
It should be kept correct and concise, usable out-of-the-box.
|
||||||
|
|
||||||
|
|
||||||
|
For more information, on how to customize this file, please see
|
||||||
|
http://lucene.apache.org/solr/guide/documents-fields-and-schema-design.html
|
||||||
|
|
||||||
|
PERFORMANCE NOTE: this schema includes many optional features and should not
|
||||||
|
be used for benchmarking. To improve performance one could
|
||||||
|
- set stored="false" for all fields possible (esp large fields) when you
|
||||||
|
only need to search on the field but don't need to return the original
|
||||||
|
value.
|
||||||
|
- set indexed="false" if you don't need to search on the field, but only
|
||||||
|
return the field as a result of searching on other indexed fields.
|
||||||
|
- remove all unneeded copyField statements
|
||||||
|
- for best index size and searching performance, set "index" to false
|
||||||
|
for all general text fields, use copyField to copy them to the
|
||||||
|
catchall "text" field, and use that for searching.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<schema name="collection1-config" version="1.6">
|
||||||
|
<!-- attribute "name" is the name of this schema and is only used for display purposes.
|
||||||
|
version="x.y" is Solr's version number for the schema syntax and
|
||||||
|
semantics. It should not normally be changed by applications.
|
||||||
|
|
||||||
|
1.0: multiValued attribute did not exist, all fields are multiValued
|
||||||
|
by nature
|
||||||
|
1.1: multiValued attribute introduced, false by default
|
||||||
|
1.2: omitTermFreqAndPositions attribute introduced, true by default
|
||||||
|
except for text fields.
|
||||||
|
1.3: removed optional field compress feature
|
||||||
|
1.4: autoGeneratePhraseQueries attribute introduced to drive QueryParser
|
||||||
|
behavior when a single string produces multiple tokens. Defaults
|
||||||
|
to off for version >= 1.4
|
||||||
|
1.5: omitNorms defaults to true for primitive field types
|
||||||
|
(int, float, boolean, string...)
|
||||||
|
1.6: useDocValuesAsStored defaults to true.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- Valid attributes for fields:
|
||||||
|
name: mandatory - the name for the field
|
||||||
|
type: mandatory - the name of a field type from the
|
||||||
|
fieldTypes section
|
||||||
|
indexed: true if this field should be indexed (searchable or sortable)
|
||||||
|
stored: true if this field should be retrievable
|
||||||
|
docValues: true if this field should have doc values. Doc Values is
|
||||||
|
recommended (required, if you are using *Point fields) for faceting,
|
||||||
|
grouping, sorting and function queries. Doc Values will make the index
|
||||||
|
faster to load, more NRT-friendly and more memory-efficient.
|
||||||
|
They are currently only supported by StrField, UUIDField, all
|
||||||
|
*PointFields, and depending on the field type, they might require
|
||||||
|
the field to be single-valued, be required or have a default value
|
||||||
|
(check the documentation of the field type you're interested in for
|
||||||
|
more information)
|
||||||
|
multiValued: true if this field may contain multiple values per document
|
||||||
|
omitNorms: (expert) set to true to omit the norms associated with
|
||||||
|
this field (this disables length normalization and index-time
|
||||||
|
boosting for the field, and saves some memory). Only full-text
|
||||||
|
fields or fields that need an index-time boost need norms.
|
||||||
|
Norms are omitted for primitive (non-analyzed) types by default.
|
||||||
|
termVectors: [false] set to true to store the term vector for a
|
||||||
|
given field.
|
||||||
|
When using MoreLikeThis, fields used for similarity should be
|
||||||
|
stored for best performance.
|
||||||
|
termPositions: Store position information with the term vector.
|
||||||
|
This will increase storage costs.
|
||||||
|
termOffsets: Store offset information with the term vector. This
|
||||||
|
will increase storage costs.
|
||||||
|
required: The field is required. It will throw an error if the
|
||||||
|
value does not exist
|
||||||
|
default: a value that should be used if no value is specified
|
||||||
|
when adding a document.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- field names should consist of alphanumeric or underscore characters only and
|
||||||
|
not start with a digit. This is not currently strictly enforced,
|
||||||
|
but other field names will not have first class support from all components
|
||||||
|
and back compatibility is not guaranteed. Names with both leading and
|
||||||
|
trailing underscores (e.g. _version_) are reserved.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- In this _default configset, only four fields are pre-declared:
|
||||||
|
id, _version_, and _text_ and _root_. All other fields will be type guessed and added via the
|
||||||
|
"add-unknown-fields-to-the-schema" update request processor chain declared in solrconfig.xml.
|
||||||
|
|
||||||
|
Note that many dynamic fields are also defined - you can use them to specify a
|
||||||
|
field's type via field naming conventions - see below.
|
||||||
|
|
||||||
|
WARNING: The _text_ catch-all field will significantly increase your index size.
|
||||||
|
If you don't need it, consider removing it and the corresponding copyField directive.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
|
||||||
|
<!-- docValues are enabled by default for long type so we don't need to index the version field -->
|
||||||
|
<field name="_version_" type="plong" indexed="false" stored="false"/>
|
||||||
|
<field name="_root_" type="string" indexed="true" stored="false" docValues="false" />
|
||||||
|
<field name="_text_" type="text_general" indexed="true" stored="false" multiValued="true"/>
|
||||||
|
|
||||||
|
<!-- This can be enabled, in case the client does not know what fields may be searched. It isn't enabled by default
|
||||||
|
because it's very expensive to index everything twice. -->
|
||||||
|
<!-- <copyField source="*" dest="_text_"/> -->
|
||||||
|
|
||||||
|
<!-- Dynamic field definitions allow using convention over configuration
|
||||||
|
for fields via the specification of patterns to match field names.
|
||||||
|
EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i)
|
||||||
|
RESTRICTION: the glob-like pattern in the name attribute must have a "*" only at the start or the end. -->
|
||||||
|
|
||||||
|
<dynamicField name="*_i" type="pint" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_is" type="pints" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_s" type="string" indexed="true" stored="true" />
|
||||||
|
<dynamicField name="*_ss" type="strings" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_l" type="plong" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_ls" type="plongs" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_txt" type="text_general" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_bs" type="booleans" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_f" type="pfloat" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_fs" type="pfloats" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_d" type="pdouble" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_ds" type="pdoubles" indexed="true" stored="true"/>
|
||||||
|
|
||||||
|
<!-- Type used for data-driven schema, to add a string copy for each text field -->
|
||||||
|
<dynamicField name="*_str" type="strings" stored="false" docValues="true" indexed="false" />
|
||||||
|
|
||||||
|
<dynamicField name="*_dt" type="pdate" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_dts" type="pdate" indexed="true" stored="true" multiValued="true"/>
|
||||||
|
<dynamicField name="*_p" type="location" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_srpt" type="location_rpt" indexed="true" stored="true"/>
|
||||||
|
|
||||||
|
<!-- payloaded dynamic fields -->
|
||||||
|
<dynamicField name="*_dpf" type="delimited_payloads_float" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_dpi" type="delimited_payloads_int" indexed="true" stored="true"/>
|
||||||
|
<dynamicField name="*_dps" type="delimited_payloads_string" indexed="true" stored="true"/>
|
||||||
|
|
||||||
|
<dynamicField name="attr_*" type="text_general" indexed="true" stored="true" multiValued="true"/>
|
||||||
|
|
||||||
|
<!-- Field to use to determine and enforce document uniqueness.
|
||||||
|
Unless this field is marked with required="false", it will be a required field
|
||||||
|
-->
|
||||||
|
<uniqueKey>id</uniqueKey>
|
||||||
|
|
||||||
|
<!-- copyField commands copy one field to another at the time a document
|
||||||
|
is added to the index. It's used either to index the same field differently,
|
||||||
|
or to add multiple fields to the same field for easier/faster searching.
|
||||||
|
|
||||||
|
<copyField source="sourceFieldName" dest="destinationFieldName"/>
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- field type definitions. The "name" attribute is
|
||||||
|
just a label to be used by field definitions. The "class"
|
||||||
|
attribute and any other attributes determine the real
|
||||||
|
behavior of the fieldType.
|
||||||
|
Class names starting with "solr" refer to java classes in a
|
||||||
|
standard package such as org.apache.solr.analysis
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- sortMissingLast and sortMissingFirst attributes are optional attributes are
|
||||||
|
currently supported on types that are sorted internally as strings
|
||||||
|
and on numeric types.
|
||||||
|
This includes "string", "boolean", "pint", "pfloat", "plong", "pdate", "pdouble".
|
||||||
|
- If sortMissingLast="true", then a sort on this field will cause documents
|
||||||
|
without the field to come after documents with the field,
|
||||||
|
regardless of the requested sort order (asc or desc).
|
||||||
|
- If sortMissingFirst="true", then a sort on this field will cause documents
|
||||||
|
without the field to come before documents with the field,
|
||||||
|
regardless of the requested sort order.
|
||||||
|
- If sortMissingLast="false" and sortMissingFirst="false" (the default),
|
||||||
|
then default lucene sorting will be used which places docs without the
|
||||||
|
field first in an ascending sort and last in a descending sort.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- The StrField type is not analyzed, but indexed/stored verbatim. -->
|
||||||
|
<fieldType name="string" class="solr.StrField" sortMissingLast="true" docValues="true" />
|
||||||
|
<fieldType name="strings" class="solr.StrField" sortMissingLast="true" multiValued="true" docValues="true" />
|
||||||
|
|
||||||
|
<!-- boolean type: "true" or "false" -->
|
||||||
|
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
|
||||||
|
<fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
|
||||||
|
|
||||||
|
<!--
|
||||||
|
Numeric field types that index values using KD-trees.
|
||||||
|
Point fields don't support FieldCache, so they must have docValues="true" if needed for sorting, faceting, functions, etc.
|
||||||
|
-->
|
||||||
|
<fieldType name="pint" class="solr.IntPointField" docValues="true"/>
|
||||||
|
<fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
|
||||||
|
<fieldType name="plong" class="solr.LongPointField" docValues="true"/>
|
||||||
|
<fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
|
||||||
|
|
||||||
|
<fieldType name="pints" class="solr.IntPointField" docValues="true" multiValued="true"/>
|
||||||
|
<fieldType name="pfloats" class="solr.FloatPointField" docValues="true" multiValued="true"/>
|
||||||
|
<fieldType name="plongs" class="solr.LongPointField" docValues="true" multiValued="true"/>
|
||||||
|
<fieldType name="pdoubles" class="solr.DoublePointField" docValues="true" multiValued="true"/>
|
||||||
|
|
||||||
|
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
|
||||||
|
is a more restricted form of the canonical representation of dateTime
|
||||||
|
http://www.w3.org/TR/xmlschema-2/#dateTime
|
||||||
|
The trailing "Z" designates UTC time and is mandatory.
|
||||||
|
Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
|
||||||
|
All other components are mandatory.
|
||||||
|
|
||||||
|
Expressions can also be used to denote calculations that should be
|
||||||
|
performed relative to "NOW" to determine the value, ie...
|
||||||
|
|
||||||
|
NOW/HOUR
|
||||||
|
... Round to the start of the current hour
|
||||||
|
NOW-1DAY
|
||||||
|
... Exactly 1 day prior to now
|
||||||
|
NOW/DAY+6MONTHS+3DAYS
|
||||||
|
... 6 months and 3 days in the future from the start of
|
||||||
|
the current day
|
||||||
|
|
||||||
|
-->
|
||||||
|
<!-- KD-tree versions of date fields -->
|
||||||
|
<fieldType name="pdate" class="solr.DatePointField" docValues="true"/>
|
||||||
|
<fieldType name="pdates" class="solr.DatePointField" docValues="true" multiValued="true"/>
|
||||||
|
|
||||||
|
<!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
|
||||||
|
<fieldType name="binary" class="solr.BinaryField"/>
|
||||||
|
|
||||||
|
<!-- solr.TextField allows the specification of custom text analyzers
|
||||||
|
specified as a tokenizer and a list of token filters. Different
|
||||||
|
analyzers may be specified for indexing and querying.
|
||||||
|
|
||||||
|
The optional positionIncrementGap puts space between multiple fields of
|
||||||
|
this type on the same document, with the purpose of preventing false phrase
|
||||||
|
matching across fields.
|
||||||
|
|
||||||
|
For more info on customizing your analyzer chain, please see
|
||||||
|
http://lucene.apache.org/solr/guide/understanding-analyzers-tokenizers-and-filters.html#understanding-analyzers-tokenizers-and-filters
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- One can also specify an existing Analyzer class that has a
|
||||||
|
default constructor via the class attribute on the analyzer element.
|
||||||
|
Example:
|
||||||
|
<fieldType name="text_greek" class="solr.TextField">
|
||||||
|
<analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
|
||||||
|
</fieldType>
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- A text field that only splits on whitespace for exact matching of words -->
|
||||||
|
<dynamicField name="*_ws" type="text_ws" indexed="true" stored="true"/>
|
||||||
|
<fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
|
||||||
|
<analyzer>
|
||||||
|
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
|
||||||
|
</analyzer>
|
||||||
|
</fieldType>
|
||||||
|
|
||||||
|
<!-- A general text field that has reasonable, generic
|
||||||
|
cross-language defaults: it tokenizes with StandardTokenizer,
|
||||||
|
removes stop words from case-insensitive "stopwords.txt"
|
||||||
|
(empty by default), and down cases. At query time only, it
|
||||||
|
also applies synonyms.
|
||||||
|
-->
|
||||||
|
<fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">
|
||||||
|
<analyzer type="index">
|
||||||
|
<tokenizer class="solr.StandardTokenizerFactory"/>
|
||||||
|
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" />
|
||||||
|
<!-- in this example, we will only use synonyms at query time
|
||||||
|
<filter class="solr.SynonymGraphFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
|
||||||
|
<filter class="solr.FlattenGraphFilterFactory"/>
|
||||||
|
-->
|
||||||
|
<filter class="solr.LowerCaseFilterFactory"/>
|
||||||
|
</analyzer>
|
||||||
|
<analyzer type="query">
|
||||||
|
<tokenizer class="solr.StandardTokenizerFactory"/>
|
||||||
|
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" />
|
||||||
|
<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
|
||||||
|
<filter class="solr.LowerCaseFilterFactory"/>
|
||||||
|
</analyzer>
|
||||||
|
</fieldType>
|
||||||
|
|
||||||
|
<!-- Just like text_general except it reverses the characters of
|
||||||
|
each token, to enable more efficient leading wildcard queries.
|
||||||
|
-->
|
||||||
|
<dynamicField name="*_txt_rev" type="text_general_rev" indexed="true" stored="true"/>
|
||||||
|
<fieldType name="text_general_rev" class="solr.TextField" positionIncrementGap="100">
|
||||||
|
<analyzer type="index">
|
||||||
|
<tokenizer class="solr.StandardTokenizerFactory"/>
|
||||||
|
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" />
|
||||||
|
<filter class="solr.LowerCaseFilterFactory"/>
|
||||||
|
<filter class="solr.ReversedWildcardFilterFactory" withOriginal="true"
|
||||||
|
maxPosAsterisk="3" maxPosQuestion="2" maxFractionAsterisk="0.33"/>
|
||||||
|
</analyzer>
|
||||||
|
<analyzer type="query">
|
||||||
|
<tokenizer class="solr.StandardTokenizerFactory"/>
|
||||||
|
<filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
|
||||||
|
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" />
|
||||||
|
<filter class="solr.LowerCaseFilterFactory"/>
|
||||||
|
</analyzer>
|
||||||
|
</fieldType>
|
||||||
|
|
||||||
|
<dynamicField name="*_phon_en" type="phonetic_en" indexed="true" stored="true"/>
|
||||||
|
<fieldType name="phonetic_en" stored="false" indexed="true" class="solr.TextField" >
|
||||||
|
<analyzer>
|
||||||
|
<tokenizer class="solr.StandardTokenizerFactory"/>
|
||||||
|
<filter class="solr.DoubleMetaphoneFilterFactory" inject="false"/>
|
||||||
|
</analyzer>
|
||||||
|
</fieldType>
|
||||||
|
|
||||||
|
<!-- lowercases the entire field value, keeping it as a single token. -->
|
||||||
|
<dynamicField name="*_s_lower" type="lowercase" indexed="true" stored="true"/>
|
||||||
|
<fieldType name="lowercase" class="solr.TextField" positionIncrementGap="100">
|
||||||
|
<analyzer>
|
||||||
|
<tokenizer class="solr.KeywordTokenizerFactory"/>
|
||||||
|
<filter class="solr.LowerCaseFilterFactory" />
|
||||||
|
</analyzer>
|
||||||
|
</fieldType>
|
||||||
|
|
||||||
|
<!--
|
||||||
|
Example of using PathHierarchyTokenizerFactory at index time, so
|
||||||
|
queries for paths match documents at that path, or in descendent paths
|
||||||
|
-->
|
||||||
|
<dynamicField name="*_descendent_path" type="descendent_path" indexed="true" stored="true"/>
|
||||||
|
<fieldType name="descendent_path" class="solr.TextField">
|
||||||
|
<analyzer type="index">
|
||||||
|
<tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
|
||||||
|
</analyzer>
|
||||||
|
<analyzer type="query">
|
||||||
|
<tokenizer class="solr.KeywordTokenizerFactory" />
|
||||||
|
</analyzer>
|
||||||
|
</fieldType>
|
||||||
|
|
||||||
|
<!--
|
||||||
|
Example of using PathHierarchyTokenizerFactory at query time, so
|
||||||
|
queries for paths match documents at that path, or in ancestor paths
|
||||||
|
-->
|
||||||
|
<dynamicField name="*_ancestor_path" type="ancestor_path" indexed="true" stored="true"/>
|
||||||
|
<fieldType name="ancestor_path" class="solr.TextField">
|
||||||
|
<analyzer type="index">
|
||||||
|
<tokenizer class="solr.KeywordTokenizerFactory" />
|
||||||
|
</analyzer>
|
||||||
|
<analyzer type="query">
|
||||||
|
<tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
|
||||||
|
</analyzer>
|
||||||
|
</fieldType>
|
||||||
|
|
||||||
|
<!-- This point type indexes the coordinates as separate fields (subFields)
|
||||||
|
If subFieldType is defined, it references a type, and a dynamic field
|
||||||
|
definition is created matching *___<typename>. Alternately, if
|
||||||
|
subFieldSuffix is defined, that is used to create the subFields.
|
||||||
|
Example: if subFieldType="double", then the coordinates would be
|
||||||
|
indexed in fields myloc_0___double,myloc_1___double.
|
||||||
|
Example: if subFieldSuffix="_d" then the coordinates would be indexed
|
||||||
|
in fields myloc_0_d,myloc_1_d
|
||||||
|
The subFields are an implementation detail of the fieldType, and end
|
||||||
|
users normally should not need to know about them.
|
||||||
|
-->
|
||||||
|
<dynamicField name="*_point" type="point" indexed="true" stored="true"/>
|
||||||
|
<fieldType name="point" class="solr.PointType" dimension="2" subFieldSuffix="_d"/>
|
||||||
|
|
||||||
|
<!-- A specialized field for geospatial search filters and distance sorting. -->
|
||||||
|
<fieldType name="location" class="solr.LatLonPointSpatialField" docValues="true"/>
|
||||||
|
|
||||||
|
<!-- A geospatial field type that supports multiValued and polygon shapes.
|
||||||
|
For more information about this and other spatial fields see:
|
||||||
|
http://lucene.apache.org/solr/guide/spatial-search.html
|
||||||
|
-->
|
||||||
|
<fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
|
||||||
|
geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers" />
|
||||||
|
|
||||||
|
<!-- Payloaded field types -->
|
||||||
|
<fieldType name="delimited_payloads_float" stored="false" indexed="true" class="solr.TextField">
|
||||||
|
<analyzer>
|
||||||
|
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
|
||||||
|
<filter class="solr.DelimitedPayloadTokenFilterFactory" encoder="float"/>
|
||||||
|
</analyzer>
|
||||||
|
</fieldType>
|
||||||
|
<fieldType name="delimited_payloads_int" stored="false" indexed="true" class="solr.TextField">
|
||||||
|
<analyzer>
|
||||||
|
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
|
||||||
|
<filter class="solr.DelimitedPayloadTokenFilterFactory" encoder="integer"/>
|
||||||
|
</analyzer>
|
||||||
|
</fieldType>
|
||||||
|
<fieldType name="delimited_payloads_string" stored="false" indexed="true" class="solr.TextField">
|
||||||
|
<analyzer>
|
||||||
|
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
|
||||||
|
<filter class="solr.DelimitedPayloadTokenFilterFactory" encoder="identity"/>
|
||||||
|
</analyzer>
|
||||||
|
</fieldType>
|
||||||
|
|
||||||
|
<!-- Similarity is the scoring routine for each document vs. a query.
|
||||||
|
A custom Similarity or SimilarityFactory may be specified here, but
|
||||||
|
the default is fine for most applications.
|
||||||
|
For more info: http://lucene.apache.org/solr/guide/other-schema-elements.html#OtherSchemaElements-Similarity
|
||||||
|
-->
|
||||||
|
<!--
|
||||||
|
<similarity class="com.example.solr.CustomSimilarityFactory">
|
||||||
|
<str name="paramkey">param value</str>
|
||||||
|
</similarity>
|
||||||
|
-->
|
||||||
|
|
||||||
|
</schema>
|
||||||
|
|
@ -0,0 +1,232 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8" ?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<config>
|
||||||
|
|
||||||
|
<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
|
||||||
|
|
||||||
|
<dataDir>${solr.data.dir:}</dataDir>
|
||||||
|
|
||||||
|
<directoryFactory name="DirectoryFactory"
|
||||||
|
class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
|
||||||
|
|
||||||
|
<codecFactory class="solr.SchemaCodecFactory"/>
|
||||||
|
|
||||||
|
<indexConfig>
|
||||||
|
<lockType>${solr.lock.type:native}</lockType>
|
||||||
|
</indexConfig>
|
||||||
|
|
||||||
|
<jmx />
|
||||||
|
|
||||||
|
<updateHandler class="solr.DirectUpdateHandler2">
|
||||||
|
|
||||||
|
<updateLog>
|
||||||
|
<str name="dir">${solr.ulog.dir:}</str>
|
||||||
|
<int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int>
|
||||||
|
</updateLog>
|
||||||
|
|
||||||
|
<autoCommit>
|
||||||
|
<maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
|
||||||
|
<openSearcher>false</openSearcher>
|
||||||
|
</autoCommit>
|
||||||
|
|
||||||
|
<autoSoftCommit>
|
||||||
|
<maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
|
||||||
|
</autoSoftCommit>
|
||||||
|
|
||||||
|
</updateHandler>
|
||||||
|
|
||||||
|
<query>
|
||||||
|
|
||||||
|
<maxBooleanClauses>1024</maxBooleanClauses>
|
||||||
|
|
||||||
|
<filterCache class="solr.FastLRUCache"
|
||||||
|
size="512"
|
||||||
|
initialSize="512"
|
||||||
|
autowarmCount="0"/>
|
||||||
|
|
||||||
|
<queryResultCache class="solr.LRUCache"
|
||||||
|
size="512"
|
||||||
|
initialSize="512"
|
||||||
|
autowarmCount="0"/>
|
||||||
|
|
||||||
|
<documentCache class="solr.LRUCache"
|
||||||
|
size="512"
|
||||||
|
initialSize="512"
|
||||||
|
autowarmCount="0"/>
|
||||||
|
|
||||||
|
<cache name="perSegFilter"
|
||||||
|
class="solr.search.LRUCache"
|
||||||
|
size="10"
|
||||||
|
initialSize="0"
|
||||||
|
autowarmCount="10"
|
||||||
|
regenerator="solr.NoOpRegenerator" />
|
||||||
|
|
||||||
|
<enableLazyFieldLoading>true</enableLazyFieldLoading>
|
||||||
|
|
||||||
|
<queryResultWindowSize>20</queryResultWindowSize>
|
||||||
|
|
||||||
|
<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
|
||||||
|
|
||||||
|
<listener event="newSearcher" class="solr.QuerySenderListener">
|
||||||
|
<arr name="queries">
|
||||||
|
</arr>
|
||||||
|
</listener>
|
||||||
|
<listener event="firstSearcher" class="solr.QuerySenderListener">
|
||||||
|
<arr name="queries">
|
||||||
|
</arr>
|
||||||
|
</listener>
|
||||||
|
|
||||||
|
<useColdSearcher>false</useColdSearcher>
|
||||||
|
|
||||||
|
</query>
|
||||||
|
|
||||||
|
<requestDispatcher>
|
||||||
|
|
||||||
|
<httpCaching never304="true" />
|
||||||
|
|
||||||
|
</requestDispatcher>
|
||||||
|
|
||||||
|
<!-- Request Handlers
|
||||||
|
|
||||||
|
http://wiki.apache.org/solr/SolrRequestHandler
|
||||||
|
|
||||||
|
Incoming queries will be dispatched to a specific handler by name
|
||||||
|
based on the path specified in the request.
|
||||||
|
|
||||||
|
If a Request Handler is declared with startup="lazy", then it will
|
||||||
|
not be initialized until the first request that uses it.
|
||||||
|
|
||||||
|
-->
|
||||||
|
<!-- SearchHandler
|
||||||
|
|
||||||
|
http://wiki.apache.org/solr/SearchHandler
|
||||||
|
|
||||||
|
For processing Search Queries, the primary Request Handler
|
||||||
|
provided with Solr is "SearchHandler" It delegates to a sequent
|
||||||
|
of SearchComponents (see below) and supports distributed
|
||||||
|
queries across multiple shards
|
||||||
|
-->
|
||||||
|
<requestHandler name="/select" class="solr.SearchHandler">
|
||||||
|
<lst name="defaults">
|
||||||
|
<str name="echoParams">explicit</str>
|
||||||
|
<int name="rows">10</int>
|
||||||
|
</lst>
|
||||||
|
</requestHandler>
|
||||||
|
|
||||||
|
<!-- Update Processors
|
||||||
|
|
||||||
|
Chains of Update Processor Factories for dealing with Update
|
||||||
|
Requests can be declared, and then used by name in Update
|
||||||
|
Request Processors
|
||||||
|
|
||||||
|
http://wiki.apache.org/solr/UpdateRequestProcessor
|
||||||
|
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- Add unknown fields to the schema
|
||||||
|
|
||||||
|
Field type guessing update processors that will
|
||||||
|
attempt to parse string-typed field values as Booleans, Longs,
|
||||||
|
Doubles, or Dates, and then add schema fields with the guessed
|
||||||
|
field types. Text content will be indexed as "text_general" as
|
||||||
|
well as a copy to a plain string version in *_str.
|
||||||
|
|
||||||
|
These require that the schema is both managed and mutable, by
|
||||||
|
declaring schemaFactory as ManagedIndexSchemaFactory, with
|
||||||
|
mutable specified as true.
|
||||||
|
|
||||||
|
See http://wiki.apache.org/solr/GuessingFieldTypes
|
||||||
|
-->
|
||||||
|
<updateProcessor class="solr.UUIDUpdateProcessorFactory" name="uuid"/>
|
||||||
|
<updateProcessor class="solr.RemoveBlankFieldUpdateProcessorFactory" name="remove-blank"/>
|
||||||
|
<updateProcessor class="solr.FieldNameMutatingUpdateProcessorFactory" name="field-name-mutating">
|
||||||
|
<str name="pattern">[^\w-\.]</str>
|
||||||
|
<str name="replacement">_</str>
|
||||||
|
</updateProcessor>
|
||||||
|
<updateProcessor class="solr.ParseBooleanFieldUpdateProcessorFactory" name="parse-boolean"/>
|
||||||
|
<updateProcessor class="solr.ParseLongFieldUpdateProcessorFactory" name="parse-long"/>
|
||||||
|
<updateProcessor class="solr.ParseDoubleFieldUpdateProcessorFactory" name="parse-double"/>
|
||||||
|
<updateProcessor class="solr.ParseDateFieldUpdateProcessorFactory" name="parse-date">
|
||||||
|
<arr name="format">
|
||||||
|
<str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
|
||||||
|
<str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
|
||||||
|
<str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
|
||||||
|
<str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
|
||||||
|
<str>yyyy-MM-dd'T'HH:mm:ssZ</str>
|
||||||
|
<str>yyyy-MM-dd'T'HH:mm:ss</str>
|
||||||
|
<str>yyyy-MM-dd'T'HH:mmZ</str>
|
||||||
|
<str>yyyy-MM-dd'T'HH:mm</str>
|
||||||
|
<str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
|
||||||
|
<str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
|
||||||
|
<str>yyyy-MM-dd HH:mm:ss.SSS</str>
|
||||||
|
<str>yyyy-MM-dd HH:mm:ss,SSS</str>
|
||||||
|
<str>yyyy-MM-dd HH:mm:ssZ</str>
|
||||||
|
<str>yyyy-MM-dd HH:mm:ss</str>
|
||||||
|
<str>yyyy-MM-dd HH:mmZ</str>
|
||||||
|
<str>yyyy-MM-dd HH:mm</str>
|
||||||
|
<str>yyyy-MM-dd</str>
|
||||||
|
</arr>
|
||||||
|
</updateProcessor>
|
||||||
|
<updateProcessor class="solr.AddSchemaFieldsUpdateProcessorFactory" name="add-schema-fields">
|
||||||
|
<lst name="typeMapping">
|
||||||
|
<str name="valueClass">java.lang.String</str>
|
||||||
|
<str name="fieldType">text_general</str>
|
||||||
|
<lst name="copyField">
|
||||||
|
<str name="dest">*_str</str>
|
||||||
|
<int name="maxChars">256</int>
|
||||||
|
</lst>
|
||||||
|
<!-- Use as default mapping instead of defaultFieldType -->
|
||||||
|
<bool name="default">true</bool>
|
||||||
|
</lst>
|
||||||
|
<lst name="typeMapping">
|
||||||
|
<str name="valueClass">java.lang.Boolean</str>
|
||||||
|
<str name="fieldType">booleans</str>
|
||||||
|
</lst>
|
||||||
|
<lst name="typeMapping">
|
||||||
|
<str name="valueClass">java.util.Date</str>
|
||||||
|
<str name="fieldType">pdates</str>
|
||||||
|
</lst>
|
||||||
|
<lst name="typeMapping">
|
||||||
|
<str name="valueClass">java.lang.Long</str>
|
||||||
|
<str name="valueClass">java.lang.Integer</str>
|
||||||
|
<str name="fieldType">plongs</str>
|
||||||
|
</lst>
|
||||||
|
<lst name="typeMapping">
|
||||||
|
<str name="valueClass">java.lang.Number</str>
|
||||||
|
<str name="fieldType">pdoubles</str>
|
||||||
|
</lst>
|
||||||
|
</updateProcessor>
|
||||||
|
|
||||||
|
<!-- The update.autoCreateFields property can be turned to false to disable schemaless mode -->
|
||||||
|
<updateRequestProcessorChain name="add-unknown-fields-to-the-schema" default="${update.autoCreateFields:true}"
|
||||||
|
processor="uuid,remove-blank,field-name-mutating,parse-boolean,parse-long,parse-double,parse-date,add-schema-fields">
|
||||||
|
<processor class="solr.LogUpdateProcessorFactory"/>
|
||||||
|
<processor class="solr.DistributedUpdateProcessorFactory"/>
|
||||||
|
<processor class="solr.RunUpdateProcessorFactory"/>
|
||||||
|
</updateRequestProcessorChain>
|
||||||
|
|
||||||
|
<queryResponseWriter name="json" class="solr.JSONResponseWriter">
|
||||||
|
<!-- For the purposes of the tutorial, JSON responses are written as
|
||||||
|
plain text so that they are easy to read in *any* browser.
|
||||||
|
If you expect a MIME type of "application/json" just remove this override.
|
||||||
|
-->
|
||||||
|
<str name="content-type">text/plain; charset=UTF-8</str>
|
||||||
|
</queryResponseWriter>
|
||||||
|
|
||||||
|
</config>
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
@ -0,0 +1,29 @@
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
#-----------------------------------------------------------------------
|
||||||
|
#some test synonym mappings unlikely to appear in real input text
|
||||||
|
aaafoo => aaabar
|
||||||
|
bbbfoo => bbbfoo bbbbar
|
||||||
|
cccfoo => cccbar cccbaz
|
||||||
|
fooaaa,baraaa,bazaaa
|
||||||
|
|
||||||
|
# Some synonym groups specific to this example
|
||||||
|
GB,gib,gigabyte,gigabytes
|
||||||
|
MB,mib,megabyte,megabytes
|
||||||
|
Television, Televisions, TV, TVs
|
||||||
|
#notice we use "gib" instead of "GiB" so any WordDelimiterGraphFilter coming
|
||||||
|
#after us won't split it into two words.
|
||||||
|
|
||||||
|
# Synonym mappings can be used for spelling correction too
|
||||||
|
pixima => pixma
|
||||||
|
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
<?xml version="1.0" encoding="GB18030"?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add>
|
||||||
|
<doc>
|
||||||
|
<field name="id">GB18030TEST</field>
|
||||||
|
<field name="name">Test with some GB18030 encoded characters</field>
|
||||||
|
<field name="features">No accents here</field>
|
||||||
|
<field name="features">这是一个功能</field>
|
||||||
|
<field name="features">This is a feature (translated)</field>
|
||||||
|
<field name="features">这份文件是很有光泽</field>
|
||||||
|
<field name="features">This document is very shiny (translated)</field>
|
||||||
|
<field name="price">0.0</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
</doc>
|
||||||
|
</add>
|
||||||
|
|
||||||
|
|
@ -0,0 +1,56 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add>
|
||||||
|
<doc>
|
||||||
|
<field name="id">SP2514N</field>
|
||||||
|
<field name="name">Samsung SpinPoint P120 SP2514N - hard drive - 250 GB - ATA-133</field>
|
||||||
|
<field name="manu">Samsung Electronics Co. Ltd.</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">samsung</field>
|
||||||
|
<field name="cat">electronics</field>
|
||||||
|
<field name="cat">hard drive</field>
|
||||||
|
<field name="features">7200RPM, 8MB cache, IDE Ultra ATA-133</field>
|
||||||
|
<field name="features">NoiseGuard, SilentSeek technology, Fluid Dynamic Bearing (FDB) motor</field>
|
||||||
|
<field name="price">92.0</field>
|
||||||
|
<field name="popularity">6</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
<field name="manufacturedate_dt">2006-02-13T15:26:37Z</field>
|
||||||
|
<!-- Near Oklahoma city -->
|
||||||
|
<field name="store">35.0752,-97.032</field>
|
||||||
|
</doc>
|
||||||
|
|
||||||
|
<doc>
|
||||||
|
<field name="id">6H500F0</field>
|
||||||
|
<field name="name">Maxtor DiamondMax 11 - hard drive - 500 GB - SATA-300</field>
|
||||||
|
<field name="manu">Maxtor Corp.</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">maxtor</field>
|
||||||
|
<field name="cat">electronics</field>
|
||||||
|
<field name="cat">hard drive</field>
|
||||||
|
<field name="features">SATA 3.0Gb/s, NCQ</field>
|
||||||
|
<field name="features">8.5ms seek</field>
|
||||||
|
<field name="features">16MB cache</field>
|
||||||
|
<field name="price">350.0</field>
|
||||||
|
<field name="popularity">6</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
<!-- Buffalo store -->
|
||||||
|
<field name="store">45.17614,-93.87341</field>
|
||||||
|
<field name="manufacturedate_dt">2006-02-13T15:26:37Z</field>
|
||||||
|
</doc>
|
||||||
|
</add>
|
||||||
|
|
||||||
|
|
@ -0,0 +1,60 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add>
|
||||||
|
|
||||||
|
<doc>
|
||||||
|
<field name="id">F8V7067-APL-KIT</field>
|
||||||
|
<field name="name">Belkin Mobile Power Cord for iPod w/ Dock</field>
|
||||||
|
<field name="manu">Belkin</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">belkin</field>
|
||||||
|
<field name="cat">electronics</field>
|
||||||
|
<field name="cat">connector</field>
|
||||||
|
<field name="features">car power adapter, white</field>
|
||||||
|
<field name="weight">4.0</field>
|
||||||
|
<field name="price">19.95</field>
|
||||||
|
<field name="popularity">1</field>
|
||||||
|
<field name="inStock">false</field>
|
||||||
|
<!-- Buffalo store -->
|
||||||
|
<field name="store">45.18014,-93.87741</field>
|
||||||
|
<field name="manufacturedate_dt">2005-08-01T16:30:25Z</field>
|
||||||
|
</doc>
|
||||||
|
|
||||||
|
<doc>
|
||||||
|
<field name="id">IW-02</field>
|
||||||
|
<field name="name">iPod & iPod Mini USB 2.0 Cable</field>
|
||||||
|
<field name="manu">Belkin</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">belkin</field>
|
||||||
|
<field name="cat">electronics</field>
|
||||||
|
<field name="cat">connector</field>
|
||||||
|
<field name="features">car power adapter for iPod, white</field>
|
||||||
|
<field name="weight">2.0</field>
|
||||||
|
<field name="price">11.50</field>
|
||||||
|
<field name="popularity">1</field>
|
||||||
|
<field name="inStock">false</field>
|
||||||
|
<!-- San Francisco store -->
|
||||||
|
<field name="store">37.7752,-122.4232</field>
|
||||||
|
<field name="manufacturedate_dt">2006-02-14T23:55:59Z</field>
|
||||||
|
</doc>
|
||||||
|
|
||||||
|
|
||||||
|
</add>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,40 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add><doc>
|
||||||
|
<field name="id">MA147LL/A</field>
|
||||||
|
<field name="name">Apple 60 GB iPod with Video Playback Black</field>
|
||||||
|
<field name="manu">Apple Computer Inc.</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">apple</field>
|
||||||
|
<field name="cat">electronics</field>
|
||||||
|
<field name="cat">music</field>
|
||||||
|
<field name="features">iTunes, Podcasts, Audiobooks</field>
|
||||||
|
<field name="features">Stores up to 15,000 songs, 25,000 photos, or 150 hours of video</field>
|
||||||
|
<field name="features">2.5-inch, 320x240 color TFT LCD display with LED backlight</field>
|
||||||
|
<field name="features">Up to 20 hours of battery life</field>
|
||||||
|
<field name="features">Plays AAC, MP3, WAV, AIFF, Audible, Apple Lossless, H.264 video</field>
|
||||||
|
<field name="features">Notes, Calendar, Phone book, Hold button, Date display, Photo wallet, Built-in games, JPEG photo playback, Upgradeable firmware, USB 2.0 compatibility, Playback speed control, Rechargeable capability, Battery level indication</field>
|
||||||
|
<field name="includes">earbud headphones, USB cable</field>
|
||||||
|
<field name="weight">5.5</field>
|
||||||
|
<field name="price">399.00</field>
|
||||||
|
<field name="popularity">10</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
<!-- Dodge City store -->
|
||||||
|
<field name="store">37.7752,-100.0232</field>
|
||||||
|
<field name="manufacturedate_dt">2005-10-12T08:00:00Z</field>
|
||||||
|
</doc></add>
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add>
|
||||||
|
<doc>
|
||||||
|
<field name="id">adata</field>
|
||||||
|
<field name="compName_s">A-Data Technology</field>
|
||||||
|
<field name="address_s">46221 Landing Parkway Fremont, CA 94538</field>
|
||||||
|
</doc>
|
||||||
|
<doc>
|
||||||
|
<field name="id">apple</field>
|
||||||
|
<field name="compName_s">Apple</field>
|
||||||
|
<field name="address_s">1 Infinite Way, Cupertino CA</field>
|
||||||
|
</doc>
|
||||||
|
<doc>
|
||||||
|
<field name="id">asus</field>
|
||||||
|
<field name="compName_s">ASUS Computer</field>
|
||||||
|
<field name="address_s">800 Corporate Way Fremont, CA 94539</field>
|
||||||
|
</doc>
|
||||||
|
<doc>
|
||||||
|
<field name="id">ati</field>
|
||||||
|
<field name="compName_s">ATI Technologies</field>
|
||||||
|
<field name="address_s">33 Commerce Valley Drive East Thornhill, ON L3T 7N6 Canada</field>
|
||||||
|
</doc>
|
||||||
|
<doc>
|
||||||
|
<field name="id">belkin</field>
|
||||||
|
<field name="compName_s">Belkin</field>
|
||||||
|
<field name="address_s">12045 E. Waterfront Drive Playa Vista, CA 90094</field>
|
||||||
|
</doc>
|
||||||
|
<doc>
|
||||||
|
<field name="id">canon</field>
|
||||||
|
<field name="compName_s">Canon, Inc.</field>
|
||||||
|
<field name="address_s">One Canon Plaza Lake Success, NY 11042</field>
|
||||||
|
</doc>
|
||||||
|
<doc>
|
||||||
|
<field name="id">corsair</field>
|
||||||
|
<field name="compName_s">Corsair Microsystems</field>
|
||||||
|
<field name="address_s">46221 Landing Parkway Fremont, CA 94538</field>
|
||||||
|
</doc>
|
||||||
|
<doc>
|
||||||
|
<field name="id">dell</field>
|
||||||
|
<field name="compName_s">Dell, Inc.</field>
|
||||||
|
<field name="address_s">One Dell Way Round Rock, Texas 78682</field>
|
||||||
|
</doc>
|
||||||
|
<doc>
|
||||||
|
<field name="id">maxtor</field>
|
||||||
|
<field name="compName_s">Maxtor Corporation</field>
|
||||||
|
<field name="address_s">920 Disc Drive Scotts Valley, CA 95066</field>
|
||||||
|
</doc>
|
||||||
|
<doc>
|
||||||
|
<field name="id">samsung</field>
|
||||||
|
<field name="compName_s">Samsung Electronics Co. Ltd.</field>
|
||||||
|
<field name="address_s">105 Challenger Rd. Ridgefield Park, NJ 07660-0511</field>
|
||||||
|
</doc>
|
||||||
|
<doc>
|
||||||
|
<field name="id">viewsonic</field>
|
||||||
|
<field name="compName_s">ViewSonic Corp</field>
|
||||||
|
<field name="address_s">381 Brea Canyon Road Walnut, CA 91789-0708</field>
|
||||||
|
</doc>
|
||||||
|
</add>
|
||||||
|
|
||||||
|
|
@ -0,0 +1,77 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add>
|
||||||
|
<doc>
|
||||||
|
<field name="id">TWINX2048-3200PRO</field>
|
||||||
|
<field name="name">CORSAIR XMS 2GB (2 x 1GB) 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) Dual Channel Kit System Memory - Retail</field>
|
||||||
|
<field name="manu">Corsair Microsystems Inc.</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">corsair</field>
|
||||||
|
<field name="cat">electronics</field>
|
||||||
|
<field name="cat">memory</field>
|
||||||
|
<field name="features">CAS latency 2, 2-3-3-6 timing, 2.75v, unbuffered, heat-spreader</field>
|
||||||
|
<field name="price">185.00</field>
|
||||||
|
<field name="popularity">5</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
<!-- San Francisco store -->
|
||||||
|
<field name="store">37.7752,-122.4232</field>
|
||||||
|
<field name="manufacturedate_dt">2006-02-13T15:26:37Z</field>
|
||||||
|
|
||||||
|
<!-- a field for testing payload tagged text via DelimitedPayloadTokenFilter -->
|
||||||
|
<field name="payloads">electronics|6.0 memory|3.0</field>
|
||||||
|
</doc>
|
||||||
|
|
||||||
|
<doc>
|
||||||
|
<field name="id">VS1GB400C3</field>
|
||||||
|
<field name="name">CORSAIR ValueSelect 1GB 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) System Memory - Retail</field>
|
||||||
|
<field name="manu">Corsair Microsystems Inc.</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">corsair</field>
|
||||||
|
<field name="cat">electronics</field>
|
||||||
|
<field name="cat">memory</field>
|
||||||
|
<field name="price">74.99</field>
|
||||||
|
<field name="popularity">7</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
<!-- Dodge City store -->
|
||||||
|
<field name="store">37.7752,-100.0232</field>
|
||||||
|
<field name="manufacturedate_dt">2006-02-13T15:26:37Z</field>
|
||||||
|
|
||||||
|
<field name="payloads">electronics|4.0 memory|2.0</field>
|
||||||
|
</doc>
|
||||||
|
|
||||||
|
<doc>
|
||||||
|
<field name="id">VDBDB1A16</field>
|
||||||
|
<field name="name">A-DATA V-Series 1GB 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) System Memory - OEM</field>
|
||||||
|
<field name="manu">A-DATA Technology Inc.</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">corsair</field>
|
||||||
|
<field name="cat">electronics</field>
|
||||||
|
<field name="cat">memory</field>
|
||||||
|
<field name="features">CAS latency 3, 2.7v</field>
|
||||||
|
<!-- note: price & popularity is missing on this one -->
|
||||||
|
<field name="popularity">0</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
<!-- Buffalo store -->
|
||||||
|
<field name="store">45.18414,-93.88141</field>
|
||||||
|
<field name="manufacturedate_dt">2006-02-13T15:26:37Z</field>
|
||||||
|
|
||||||
|
<field name="payloads">electronics|0.9 memory|0.1</field>
|
||||||
|
</doc>
|
||||||
|
|
||||||
|
</add>
|
||||||
|
|
||||||
|
|
@ -0,0 +1,65 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- Example documents utilizing the CurrencyField type -->
|
||||||
|
<add>
|
||||||
|
<doc>
|
||||||
|
<field name="id">USD</field>
|
||||||
|
<field name="name">One Dollar</field>
|
||||||
|
<field name="manu">Bank of America</field>
|
||||||
|
<field name="manu_id_s">boa</field>
|
||||||
|
<field name="cat">currency</field>
|
||||||
|
<field name="features">Coins and notes</field>
|
||||||
|
<field name="price_c">1,USD</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
</doc>
|
||||||
|
|
||||||
|
<doc>
|
||||||
|
<field name="id">EUR</field>
|
||||||
|
<field name="name">One Euro</field>
|
||||||
|
<field name="manu">European Union</field>
|
||||||
|
<field name="manu_id_s">eu</field>
|
||||||
|
<field name="cat">currency</field>
|
||||||
|
<field name="features">Coins and notes</field>
|
||||||
|
<field name="price_c">1,EUR</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
</doc>
|
||||||
|
|
||||||
|
<doc>
|
||||||
|
<field name="id">GBP</field>
|
||||||
|
<field name="name">One British Pound</field>
|
||||||
|
<field name="manu">U.K.</field>
|
||||||
|
<field name="manu_id_s">uk</field>
|
||||||
|
<field name="cat">currency</field>
|
||||||
|
<field name="features">Coins and notes</field>
|
||||||
|
<field name="price_c">1,GBP</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
</doc>
|
||||||
|
|
||||||
|
<doc>
|
||||||
|
<field name="id">NOK</field>
|
||||||
|
<field name="name">One Krone</field>
|
||||||
|
<field name="manu">Bank of Norway</field>
|
||||||
|
<field name="manu_id_s">nor</field>
|
||||||
|
<field name="cat">currency</field>
|
||||||
|
<field name="features">Coins and notes</field>
|
||||||
|
<field name="price_c">1,NOK</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
</doc>
|
||||||
|
|
||||||
|
</add>
|
||||||
|
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add><doc>
|
||||||
|
<field name="id">3007WFP</field>
|
||||||
|
<field name="name">Dell Widescreen UltraSharp 3007WFP</field>
|
||||||
|
<field name="manu">Dell, Inc.</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">dell</field>
|
||||||
|
<field name="cat">electronics and computer1</field>
|
||||||
|
<field name="features">30" TFT active matrix LCD, 2560 x 1600, .25mm dot pitch, 700:1 contrast</field>
|
||||||
|
<field name="includes">USB cable</field>
|
||||||
|
<field name="weight">401.6</field>
|
||||||
|
<field name="price">2199.0</field>
|
||||||
|
<field name="popularity">6</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
<!-- Buffalo store -->
|
||||||
|
<field name="store">43.17614,-90.57341</field>
|
||||||
|
</doc></add>
|
||||||
|
|
||||||
|
|
@ -0,0 +1,33 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add><doc>
|
||||||
|
<field name="id">VA902B</field>
|
||||||
|
<field name="name">ViewSonic VA902B - flat panel display - TFT - 19"</field>
|
||||||
|
<field name="manu">ViewSonic Corp.</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">viewsonic</field>
|
||||||
|
<field name="cat">electronics and stuff2</field>
|
||||||
|
<field name="features">19" TFT active matrix LCD, 8ms response time, 1280 x 1024 native resolution</field>
|
||||||
|
<field name="weight">190.4</field>
|
||||||
|
<field name="price">279.95</field>
|
||||||
|
<field name="popularity">6</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
<!-- Buffalo store -->
|
||||||
|
<field name="store">45.18814,-93.88541</field>
|
||||||
|
</doc></add>
|
||||||
|
|
||||||
|
|
@ -0,0 +1,43 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add><doc>
|
||||||
|
<field name="id">0579B002</field>
|
||||||
|
<field name="name">Canon PIXMA MP500 All-In-One Photo Printer</field>
|
||||||
|
<field name="manu">Canon Inc.</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">canon</field>
|
||||||
|
<field name="cat">electronics</field>
|
||||||
|
<field name="cat">multifunction printer</field>
|
||||||
|
<field name="cat">printer</field>
|
||||||
|
<field name="cat">scanner</field>
|
||||||
|
<field name="cat">copier</field>
|
||||||
|
<field name="features">Multifunction ink-jet color photo printer</field>
|
||||||
|
<field name="features">Flatbed scanner, optical scan resolution of 1,200 x 2,400 dpi</field>
|
||||||
|
<field name="features">2.5" color LCD preview screen</field>
|
||||||
|
<field name="features">Duplex Copying</field>
|
||||||
|
<field name="features">Printing speed up to 29ppm black, 19ppm color</field>
|
||||||
|
<field name="features">Hi-Speed USB</field>
|
||||||
|
<field name="features">memory card: CompactFlash, Micro Drive, SmartMedia, Memory Stick, Memory Stick Pro, SD Card, and MultiMediaCard</field>
|
||||||
|
<field name="weight">352.0</field>
|
||||||
|
<field name="price">179.99</field>
|
||||||
|
<field name="popularity">6</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
<!-- Buffalo store -->
|
||||||
|
<field name="store">45.19214,-93.89941</field>
|
||||||
|
</doc></add>
|
||||||
|
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Welcome to Solr</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<p>
|
||||||
|
Here is some text
|
||||||
|
</p>
|
||||||
|
<p>distinct<br/>words</p>
|
||||||
|
<div>Here is some text in a div</div>
|
||||||
|
<div>This has a <a href="http://www.apache.org">link</a>.</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add><doc>
|
||||||
|
<field name="id">9885A004</field>
|
||||||
|
<field name="name">Canon PowerShot SD500</field>
|
||||||
|
<field name="manu">Canon Inc.</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">canon</field>
|
||||||
|
<field name="cat">electronics</field>
|
||||||
|
<field name="cat">camera</field>
|
||||||
|
<field name="features">3x zoop, 7.1 megapixel Digital ELPH</field>
|
||||||
|
<field name="features">movie clips up to 640x480 @30 fps</field>
|
||||||
|
<field name="features">2.0" TFT LCD, 118,000 pixels</field>
|
||||||
|
<field name="features">built in flash, red-eye reduction</field>
|
||||||
|
<field name="includes">32MB SD card, USB cable, AV cable, battery</field>
|
||||||
|
<field name="weight">6.4</field>
|
||||||
|
<field name="price">329.95</field>
|
||||||
|
<field name="popularity">7</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
<field name="manufacturedate_dt">2006-02-13T15:26:37Z</field>
|
||||||
|
<!-- Buffalo store -->
|
||||||
|
<field name="store">45.19614,-93.90341</field>
|
||||||
|
</doc></add>
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add>
|
||||||
|
<doc>
|
||||||
|
<field name="id">SOLR1000</field>
|
||||||
|
<field name="name">Solr, the Enterprise Search Server</field>
|
||||||
|
<field name="manu">Apache Software Foundation</field>
|
||||||
|
<field name="cat">software</field>
|
||||||
|
<field name="cat">search</field>
|
||||||
|
<field name="features">Advanced Full-Text Search Capabilities using Lucene</field>
|
||||||
|
<field name="features">Optimized for High Volume Web Traffic</field>
|
||||||
|
<field name="features">Standards Based Open Interfaces - XML and HTTP</field>
|
||||||
|
<field name="features">Comprehensive HTML Administration Interfaces</field>
|
||||||
|
<field name="features">Scalability - Efficient Replication to other Solr Search Servers</field>
|
||||||
|
<field name="features">Flexible and Adaptable with XML configuration and Schema</field>
|
||||||
|
<field name="features">Good unicode support: héllo (hello with an accent over the e)</field>
|
||||||
|
<field name="price">0.0</field>
|
||||||
|
<field name="popularity">10</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
<field name="incubationdate_dt">2006-01-17T00:00:00.000Z</field>
|
||||||
|
</doc>
|
||||||
|
</add>
|
||||||
|
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!--
|
||||||
|
After posting this to Solr with bin/post, searching for "êâîôû" from
|
||||||
|
the solr/admin/ search page must return this document.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add>
|
||||||
|
<doc>
|
||||||
|
<field name="id">UTF8TEST</field>
|
||||||
|
<field name="name">Test with some UTF-8 encoded characters</field>
|
||||||
|
<field name="manu">Apache Software Foundation</field>
|
||||||
|
<field name="cat">software</field>
|
||||||
|
<field name="cat">search</field>
|
||||||
|
<field name="features">No accents here</field>
|
||||||
|
<field name="features">This is an e acute: é</field>
|
||||||
|
<field name="features">eaiou with circumflexes: êâîôû</field>
|
||||||
|
<field name="features">eaiou with umlauts: ëäïöü</field>
|
||||||
|
<field name="features">tag with escaped chars: <nicetag/></field>
|
||||||
|
<field name="features">escaped ampersand: Bonnie & Clyde</field>
|
||||||
|
<field name="features">Outside the BMP:𐌈 codepoint=10308, a circle with an x inside. UTF8=f0908c88 UTF16=d800 df08</field>
|
||||||
|
<field name="price">0.0</field>
|
||||||
|
<field name="inStock">true</field>
|
||||||
|
</doc>
|
||||||
|
</add>
|
||||||
|
|
||||||
|
|
@ -0,0 +1,62 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<add>
|
||||||
|
<doc>
|
||||||
|
<field name="id">EN7800GTX/2DHTV/256M</field>
|
||||||
|
<field name="name">ASUS Extreme N7800GTX/2DHTV (256 MB)</field>
|
||||||
|
<!-- Denormalized -->
|
||||||
|
<field name="manu">ASUS Computer Inc.</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">asus</field>
|
||||||
|
<field name="cat">electronics</field>
|
||||||
|
<field name="cat">graphics card</field>
|
||||||
|
<field name="features">NVIDIA GeForce 7800 GTX GPU/VPU clocked at 486MHz</field>
|
||||||
|
<field name="features">256MB GDDR3 Memory clocked at 1.35GHz</field>
|
||||||
|
<field name="features">PCI Express x16</field>
|
||||||
|
<field name="features">Dual DVI connectors, HDTV out, video input</field>
|
||||||
|
<field name="features">OpenGL 2.0, DirectX 9.0</field>
|
||||||
|
<field name="weight">16.0</field>
|
||||||
|
<field name="price">479.95</field>
|
||||||
|
<field name="popularity">7</field>
|
||||||
|
<field name="store">40.7143,-74.006</field>
|
||||||
|
<field name="inStock">false</field>
|
||||||
|
<field name="manufacturedate_dt">2006-02-13T15:26:37Z/DAY</field>
|
||||||
|
</doc>
|
||||||
|
<!-- yes, you can add more than one document at a time -->
|
||||||
|
<doc>
|
||||||
|
<field name="id">100-435805</field>
|
||||||
|
<field name="name">ATI Radeon X1900 XTX 512 MB PCIE Video Card</field>
|
||||||
|
<field name="manu">ATI Technologies</field>
|
||||||
|
<!-- Join -->
|
||||||
|
<field name="manu_id_s">ati</field>
|
||||||
|
<field name="cat">electronics</field>
|
||||||
|
<field name="cat">graphics card</field>
|
||||||
|
<field name="features">ATI RADEON X1900 GPU/VPU clocked at 650MHz</field>
|
||||||
|
<field name="features">512MB GDDR3 SDRAM clocked at 1.55GHz</field>
|
||||||
|
<field name="features">PCI Express x16</field>
|
||||||
|
<field name="features">dual DVI, HDTV, svideo, composite out</field>
|
||||||
|
<field name="features">OpenGL 2.0, DirectX 9.0</field>
|
||||||
|
<field name="weight">48.0</field>
|
||||||
|
<field name="price">649.99</field>
|
||||||
|
<field name="popularity">7</field>
|
||||||
|
<field name="inStock">false</field>
|
||||||
|
<field name="manufacturedate_dt">2006-02-13T15:26:37Z/DAY</field>
|
||||||
|
<!-- NYC store -->
|
||||||
|
<field name="store">40.7143,-74.006</field>
|
||||||
|
</doc>
|
||||||
|
</add>
|
||||||
|
|
@ -0,0 +1,93 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.prometheus.collector;
|
||||||
|
|
||||||
|
import org.apache.solr.prometheus.collector.config.SolrCollectorConfig;
|
||||||
|
import org.apache.solr.prometheus.exporter.SolrExporter;
|
||||||
|
import org.apache.solr.prometheus.exporter.SolrExporterTestBase;
|
||||||
|
import io.prometheus.client.CollectorRegistry;
|
||||||
|
import org.apache.lucene.util.LuceneTestCase.Slow;
|
||||||
|
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
||||||
|
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.yaml.snakeyaml.Yaml;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileReader;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unit test for SolrCollector.
|
||||||
|
*/
|
||||||
|
@Slow
|
||||||
|
public class SolrCollectorTest extends SolrExporterTestBase {
|
||||||
|
CollectorRegistry registry;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setUp() throws Exception {
|
||||||
|
super.setUp();
|
||||||
|
registry = new CollectorRegistry();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void tearDown() throws Exception {
|
||||||
|
super.tearDown();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSolrCollector() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
CloudSolrClient cloudSolrClient = cluster.getSolrClient();
|
||||||
|
SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
SolrCollector collector = new SolrCollector(cloudSolrClient, collectorConfig, 1);
|
||||||
|
|
||||||
|
assertNotNull(collector);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCollect() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
CloudSolrClient cloudSolrClient = cluster.getSolrClient();
|
||||||
|
SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
SolrCollector collector = new SolrCollector(cloudSolrClient, collectorConfig, 1);
|
||||||
|
|
||||||
|
this.registry.register(collector);
|
||||||
|
this.registry.register(SolrExporter.scrapeErrorTotal);
|
||||||
|
|
||||||
|
// index sample docs
|
||||||
|
File exampleDocsDir = new File(getFile("exampledocs").getAbsolutePath());
|
||||||
|
List<File> xmlFiles = Arrays.asList(exampleDocsDir.listFiles((dir, name) -> name.endsWith(".xml")));
|
||||||
|
for (File xml : xmlFiles) {
|
||||||
|
ContentStreamUpdateRequest req = new ContentStreamUpdateRequest("/update");
|
||||||
|
req.addFile(xml, "application/xml");
|
||||||
|
cloudSolrClient.request(req, "collection1");
|
||||||
|
}
|
||||||
|
cloudSolrClient.commit("collection1");
|
||||||
|
|
||||||
|
// collect metrics
|
||||||
|
collector.collect();
|
||||||
|
|
||||||
|
// check scrape error count
|
||||||
|
assertEquals(0.0, registry.getSampleValue("solr_exporter_scrape_error_total", new String[]{}, new String[]{}), .001);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -0,0 +1,106 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.prometheus.collector.config;
|
||||||
|
|
||||||
|
import org.apache.solr.prometheus.scraper.config.SolrScraperConfig;
|
||||||
|
import org.apache.solr.SolrTestCaseJ4;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.yaml.snakeyaml.Yaml;
|
||||||
|
|
||||||
|
import java.io.FileReader;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unit test for SolrCollectorConfig.
|
||||||
|
*/
|
||||||
|
public class SolrCollectorConfigTest extends SolrTestCaseJ4 {
|
||||||
|
@Test
|
||||||
|
public void testCollectorConfig() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
assertNotNull(collectorConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetMetricsConfig() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
assertNotNull(collectorConfig.getMetrics());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetMetricsConfig() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
SolrScraperConfig metricsConfig = new SolrScraperConfig();
|
||||||
|
|
||||||
|
collectorConfig.setMetrics(metricsConfig);
|
||||||
|
|
||||||
|
assertNotNull(collectorConfig.getMetrics());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetCollectionsConfig() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
assertNotNull(collectorConfig.getCollections());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetCollectionsConfig() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
SolrScraperConfig collectionsConfig = new SolrScraperConfig();
|
||||||
|
|
||||||
|
collectorConfig.setCollections(collectionsConfig);
|
||||||
|
|
||||||
|
assertNotNull(collectorConfig.getCollections());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetQueryConfigs() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
assertNotNull(collectorConfig.getQueries());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetQueryConfigs() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
List<SolrScraperConfig> queryConfigs = new ArrayList<>();
|
||||||
|
|
||||||
|
collectorConfig.setQueries(queryConfigs);
|
||||||
|
|
||||||
|
assertNotNull(collectorConfig.getQueries());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,106 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.prometheus.exporter;
|
||||||
|
|
||||||
|
import org.apache.solr.prometheus.collector.config.SolrCollectorConfig;
|
||||||
|
import org.apache.http.HttpStatus;
|
||||||
|
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||||
|
import org.apache.http.client.methods.HttpGet;
|
||||||
|
import org.apache.http.impl.client.CloseableHttpClient;
|
||||||
|
import org.apache.http.impl.client.HttpClients;
|
||||||
|
import org.apache.lucene.util.LuceneTestCase.Slow;
|
||||||
|
import org.apache.solr.client.solrj.impl.CloudSolrClient;
|
||||||
|
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.yaml.snakeyaml.Yaml;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileReader;
|
||||||
|
import java.net.ServerSocket;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unit test for SolrExporter.
|
||||||
|
*/
|
||||||
|
@Slow
|
||||||
|
public class SolrExporterTest extends SolrExporterTestBase {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setUp() throws Exception {
|
||||||
|
super.setUp();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void tearDown() throws Exception {
|
||||||
|
super.tearDown();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testExecute() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
// solr client
|
||||||
|
CloudSolrClient cloudSolrClient = cluster.getSolrClient();
|
||||||
|
|
||||||
|
int port;
|
||||||
|
ServerSocket socket = null;
|
||||||
|
try {
|
||||||
|
socket = new ServerSocket(0);
|
||||||
|
port = socket.getLocalPort();
|
||||||
|
} finally {
|
||||||
|
socket.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
// index sample docs
|
||||||
|
File exampleDocsDir = new File(getFile("exampledocs").getAbsolutePath());
|
||||||
|
List<File> xmlFiles = Arrays.asList(exampleDocsDir.listFiles((dir, name) -> name.endsWith(".xml")));
|
||||||
|
for (File xml : xmlFiles) {
|
||||||
|
ContentStreamUpdateRequest req = new ContentStreamUpdateRequest("/update");
|
||||||
|
req.addFile(xml, "application/xml");
|
||||||
|
cloudSolrClient.request(req, "collection1");
|
||||||
|
}
|
||||||
|
cloudSolrClient.commit("collection1");
|
||||||
|
|
||||||
|
// start exporter
|
||||||
|
SolrExporter solrExporter = new SolrExporter(port, cloudSolrClient, collectorConfig, 1);
|
||||||
|
try {
|
||||||
|
solrExporter.start();
|
||||||
|
|
||||||
|
URI uri = new URI("http://localhost:" + String.valueOf(port) + "/metrics");
|
||||||
|
|
||||||
|
CloseableHttpClient httpclient = HttpClients.createDefault();
|
||||||
|
CloseableHttpResponse response = null;
|
||||||
|
try {
|
||||||
|
HttpGet request = new HttpGet(uri);
|
||||||
|
response = httpclient.execute(request);
|
||||||
|
|
||||||
|
int expectedHTTPStatusCode = HttpStatus.SC_OK;
|
||||||
|
int actualHTTPStatusCode = response.getStatusLine().getStatusCode();
|
||||||
|
assertEquals(expectedHTTPStatusCode, actualHTTPStatusCode);
|
||||||
|
} finally {
|
||||||
|
response.close();
|
||||||
|
httpclient.close();
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
solrExporter.stop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,53 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.prometheus.exporter;
|
||||||
|
|
||||||
|
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
|
||||||
|
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||||
|
import org.apache.solr.cloud.AbstractDistribZkTestBase;
|
||||||
|
import org.apache.solr.cloud.SolrCloudTestCase;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test base class.
|
||||||
|
*/
|
||||||
|
@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
|
||||||
|
public class SolrExporterTestBase extends SolrCloudTestCase {
|
||||||
|
public static String COLLECTION = "collection1";
|
||||||
|
public static String CONF_NAME = COLLECTION + "_config";
|
||||||
|
public static String CONF_DIR = getFile("configsets/" + COLLECTION + "/conf").getAbsolutePath();
|
||||||
|
public static int NUM_SHARDS = 2;
|
||||||
|
public static int NUM_REPLICAS = 2;
|
||||||
|
public static int MAX_SHARDS_PER_NODE = 1;
|
||||||
|
public static int NUM_NODES = (NUM_SHARDS * NUM_REPLICAS + (MAX_SHARDS_PER_NODE - 1)) / MAX_SHARDS_PER_NODE;
|
||||||
|
public static int TIMEOUT = 60;
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void setupCluster() throws Exception {
|
||||||
|
configureCluster(NUM_NODES)
|
||||||
|
.addConfig(CONF_NAME, getFile(CONF_DIR).toPath())
|
||||||
|
.configure();
|
||||||
|
|
||||||
|
CollectionAdminRequest
|
||||||
|
.createCollection(COLLECTION, CONF_NAME, NUM_SHARDS, NUM_REPLICAS)
|
||||||
|
.setMaxShardsPerNode(MAX_SHARDS_PER_NODE)
|
||||||
|
.process(cluster.getSolrClient());
|
||||||
|
|
||||||
|
AbstractDistribZkTestBase
|
||||||
|
.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(), true, true, TIMEOUT);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,121 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.prometheus.scraper.config;
|
||||||
|
|
||||||
|
import org.apache.solr.SolrTestCaseJ4;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unit test for SolrQueryConfig.
|
||||||
|
*/
|
||||||
|
public class SolrQueryConfigTest extends SolrTestCaseJ4 {
|
||||||
|
@Test
|
||||||
|
public void testQueryConfig() throws Exception {
|
||||||
|
SolrQueryConfig queryConfig = new SolrQueryConfig();
|
||||||
|
|
||||||
|
assertNotNull(queryConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetCollection() throws Exception {
|
||||||
|
SolrQueryConfig queryConfig = new SolrQueryConfig();
|
||||||
|
|
||||||
|
String expected = "";
|
||||||
|
String actual = queryConfig.getCollection();
|
||||||
|
assertEquals(expected, actual);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetCollection() throws Exception {
|
||||||
|
SolrQueryConfig queryConfig = new SolrQueryConfig();
|
||||||
|
|
||||||
|
queryConfig.setCollection("collection1");
|
||||||
|
|
||||||
|
String expected = "collection1";
|
||||||
|
String actual = queryConfig.getCollection();
|
||||||
|
assertEquals(expected, actual);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetPath() throws Exception {
|
||||||
|
SolrQueryConfig queryConfig = new SolrQueryConfig();
|
||||||
|
|
||||||
|
String expected = "";
|
||||||
|
String actual = queryConfig.getPath();
|
||||||
|
assertEquals(expected, actual);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetPath() throws Exception {
|
||||||
|
SolrQueryConfig queryConfig = new SolrQueryConfig();
|
||||||
|
|
||||||
|
queryConfig.setPath("/select");
|
||||||
|
|
||||||
|
String expected = "/select";
|
||||||
|
String actual = queryConfig.getPath();
|
||||||
|
assertEquals(expected, actual);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetParams() throws Exception {
|
||||||
|
SolrQueryConfig queryConfig = new SolrQueryConfig();
|
||||||
|
|
||||||
|
List<LinkedHashMap<String, String>> expected = new ArrayList<>();
|
||||||
|
List<LinkedHashMap<String, String>> actual = queryConfig.getParams();
|
||||||
|
assertEquals(expected, actual);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetParams() throws Exception {
|
||||||
|
SolrQueryConfig queryConfig = new SolrQueryConfig();
|
||||||
|
|
||||||
|
LinkedHashMap<String,String> param1 = new LinkedHashMap<>();
|
||||||
|
param1.put("q", "*:*");
|
||||||
|
|
||||||
|
LinkedHashMap<String,String> param2 = new LinkedHashMap<>();
|
||||||
|
param2.put("facet", "on");
|
||||||
|
|
||||||
|
queryConfig.setParams(Arrays.asList(param1, param2));
|
||||||
|
|
||||||
|
List<LinkedHashMap<String, String>> expected = Arrays.asList(param1, param2);
|
||||||
|
List<LinkedHashMap<String, String>> actual = queryConfig.getParams();
|
||||||
|
assertEquals(expected, actual);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetParamsString() throws Exception {
|
||||||
|
SolrQueryConfig queryConfig = new SolrQueryConfig();
|
||||||
|
|
||||||
|
LinkedHashMap<String,String> param1 = new LinkedHashMap<>();
|
||||||
|
param1.put("q", "*:*");
|
||||||
|
param1.put("fq", "manu:apple");
|
||||||
|
|
||||||
|
LinkedHashMap<String,String> param2 = new LinkedHashMap<>();
|
||||||
|
param2.put("facet", "on");
|
||||||
|
|
||||||
|
queryConfig.setParams(Arrays.asList(param1, param2));
|
||||||
|
|
||||||
|
String expected = "q=*:*&fq=manu:apple&facet=on";
|
||||||
|
String actual = queryConfig.getParamsString();
|
||||||
|
assertEquals(expected, actual);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,86 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.prometheus.scraper.config;
|
||||||
|
|
||||||
|
import org.apache.solr.prometheus.collector.config.SolrCollectorConfig;
|
||||||
|
import org.apache.solr.SolrTestCaseJ4;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.yaml.snakeyaml.Yaml;
|
||||||
|
|
||||||
|
import java.io.FileReader;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unit test for SolrScraperConfig.
|
||||||
|
*/
|
||||||
|
public class SolrScraperConfigTest extends SolrTestCaseJ4 {
|
||||||
|
@Test
|
||||||
|
public void testScraperConfig() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
SolrCollectorConfig config = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
SolrScraperConfig scraperConfig = config.getMetrics();
|
||||||
|
|
||||||
|
assertNotNull(scraperConfig);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetJsonQueries() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
SolrScraperConfig scraperConfig = collectorConfig.getMetrics();
|
||||||
|
|
||||||
|
assertNotNull(scraperConfig.getJsonQueries());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetJsonQueries() throws Exception {
|
||||||
|
List<String> jsonQueries = new ArrayList<>();
|
||||||
|
|
||||||
|
SolrScraperConfig scraperConfig = new SolrScraperConfig();
|
||||||
|
|
||||||
|
scraperConfig.setJsonQueries(jsonQueries);
|
||||||
|
|
||||||
|
assertNotNull(scraperConfig.getJsonQueries());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetQueryConfig() throws Exception {
|
||||||
|
String configFile = getFile("conf/config.yml").getAbsolutePath();
|
||||||
|
|
||||||
|
SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile), SolrCollectorConfig.class);
|
||||||
|
|
||||||
|
SolrScraperConfig scraperConfig = collectorConfig.getMetrics();
|
||||||
|
|
||||||
|
assertNotNull(scraperConfig.getQuery());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSetQueryConfig() throws Exception {
|
||||||
|
SolrQueryConfig queryConfig = new SolrQueryConfig();
|
||||||
|
|
||||||
|
SolrScraperConfig scraperConfig = new SolrScraperConfig();
|
||||||
|
|
||||||
|
scraperConfig.setQuery(queryConfig);
|
||||||
|
|
||||||
|
assertNotNull(scraperConfig.getQuery());
|
||||||
|
}
|
||||||
|
}
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 790 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
|
|
@ -0,0 +1,250 @@
|
||||||
|
= Monitoring Solr with Prometheus and Grafana
|
||||||
|
// Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
// or more contributor license agreements. See the NOTICE file
|
||||||
|
// distributed with this work for additional information
|
||||||
|
// regarding copyright ownership. The ASF licenses this file
|
||||||
|
// to you under the Apache License, Version 2.0 (the
|
||||||
|
// "License"); you may not use this file except in compliance
|
||||||
|
// with the License. You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing,
|
||||||
|
// software distributed under the License is distributed on an
|
||||||
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
// KIND, either express or implied. See the License for the
|
||||||
|
// specific language governing permissions and limitations
|
||||||
|
// under the License.
|
||||||
|
|
||||||
|
You can monitor Solr using solr-exporter that exposes Solr's metrics to https://prometheus.io[Prometheus], and visualize metrics using https://grafana.com[Grafana].
|
||||||
|
|
||||||
|
It allows users to monitor not only Solr metrics which come from <<metrics-reporting.adoc#metrics-api,Metrics API>> but also facet counts which come from <<searching.adoc#searching,Searching>>.
|
||||||
|
|
||||||
|
.solr-exporter Diagram
|
||||||
|
image::images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png[image,width=600]
|
||||||
|
|
||||||
|
This feature is experimental status.
|
||||||
|
|
||||||
|
== Running solr-exporter
|
||||||
|
|
||||||
|
You can start solr-exporter by running `./bin/solr-exporter` from the solr-exporter directory.
|
||||||
|
|
||||||
|
[source,plain]
|
||||||
|
----
|
||||||
|
$ cd ./contrib/prometheus-exporter
|
||||||
|
$ ./bin/solr-exporter -p 9983 -b http://localhost:8983/solr -f ./conf/config.yml -n 8
|
||||||
|
----
|
||||||
|
|
||||||
|
If you are on Windows platform, you can start solr-exporter by running `.\bin\solr-exporter.cmd` instead.
|
||||||
|
|
||||||
|
[source,plain]
|
||||||
|
----
|
||||||
|
> cd .\contrib\prometheus
|
||||||
|
> .\bin\solr-exporter.cmd -p 9983 -b http://localhost:8983/solr -f .\conf\config.yml -n 8
|
||||||
|
----
|
||||||
|
|
||||||
|
You can also connect to Solr in SolrCloud mode like this.
|
||||||
|
|
||||||
|
[source,plain]
|
||||||
|
----
|
||||||
|
$ cd ./contrib/prometheus
|
||||||
|
$ ./bin/solr-exporter -p 9983 -z localhost:2181/solr -f ./conf/config.yml -n 16
|
||||||
|
----
|
||||||
|
|
||||||
|
See command help:
|
||||||
|
|
||||||
|
[source,plain]
|
||||||
|
----
|
||||||
|
$ ./bin/solr-exporter -h
|
||||||
|
usage: SolrCollector [-h] [-v] [-p PORT] [-b BASE_URL] [-z ZK_HOST] [-f CONFIG]
|
||||||
|
[-n NUM_THREADS]
|
||||||
|
|
||||||
|
Prometheus exporter for Apache Solr.
|
||||||
|
|
||||||
|
optional arguments:
|
||||||
|
-h, --help show this help message and exit
|
||||||
|
-p PORT, --port PORT solr-exporter listen port
|
||||||
|
-b BASE_URL, --baseurl BASE_URL
|
||||||
|
specify Solr base URL when connecting to Solr in standalone mode (for
|
||||||
|
example 'http://localhost:8983/solr')
|
||||||
|
-z ZK_HOST, --zkhost ZK_HOST
|
||||||
|
specify ZooKeeper connection string when connecting to Solr in
|
||||||
|
SolrCloud mode (for example 'localhost:2181/solr')
|
||||||
|
-f CONFIG, --config-file CONFIG
|
||||||
|
specify configuration file
|
||||||
|
-n NUM_THREADS, --num-thread NUM_THREADS
|
||||||
|
specify number of threads
|
||||||
|
----
|
||||||
|
|
||||||
|
The Solr's metrics exposed by solr-exporter can see at the following URL.
|
||||||
|
|
||||||
|
http://localhost:9983/metrics[http://localhost:9983/metrics]
|
||||||
|
|
||||||
|
|
||||||
|
== Configuration
|
||||||
|
|
||||||
|
The configuration is in `./config/config.yml`. An example with all possible options:
|
||||||
|
|
||||||
|
[source,plain]
|
||||||
|
----
|
||||||
|
ping:
|
||||||
|
query:
|
||||||
|
path: /admin/ping
|
||||||
|
jsonQueries:
|
||||||
|
- |-
|
||||||
|
. as $object | $object |
|
||||||
|
(if $object.status == "OK" then 1.0 else 0.0 end) as $value |
|
||||||
|
{
|
||||||
|
name : "solr_ping",
|
||||||
|
type : "GAUGE",
|
||||||
|
help : "See following URL: https://lucene.apache.org/solr/guide/ping.html",
|
||||||
|
label_names : [],
|
||||||
|
label_values : [],
|
||||||
|
value : $value
|
||||||
|
}
|
||||||
|
|
||||||
|
metrics:
|
||||||
|
query:
|
||||||
|
path: /admin/metrics
|
||||||
|
params:
|
||||||
|
- group: 'all'
|
||||||
|
- type: 'all'
|
||||||
|
- prefix: ''
|
||||||
|
- property: ''
|
||||||
|
jsonQueries:
|
||||||
|
# solr_metrics_jetty_response_count
|
||||||
|
- |-
|
||||||
|
.metrics["solr.jetty"] | to_entries | .[] | select(.key | startswith("org.eclipse.jetty.server.handler.DefaultHandler")) | select(.key | endswith("xx-responses")) as $object |
|
||||||
|
$object.key | split(".") | last | split("-") | first as $status |
|
||||||
|
$object.value.count as $value |
|
||||||
|
{
|
||||||
|
name : "solr_metrics_jetty_response_count",
|
||||||
|
type : "gauge",
|
||||||
|
help : "See following URL: https://lucene.apache.org/solr/guide/metrics-reporting.html",
|
||||||
|
label_names : ["status"],
|
||||||
|
label_values : [$status],
|
||||||
|
value : $value
|
||||||
|
}
|
||||||
|
|
||||||
|
...
|
||||||
|
|
||||||
|
collections:
|
||||||
|
query:
|
||||||
|
path: /admin/collections
|
||||||
|
params:
|
||||||
|
- action: 'CLUSTERSTATUS'
|
||||||
|
jsonQueries:
|
||||||
|
# solr_collections_cluster_status_live_nodes
|
||||||
|
- |-
|
||||||
|
.cluster.live_nodes | length as $value|
|
||||||
|
{
|
||||||
|
name : "solr_collections_cluster_status_live_nodes",
|
||||||
|
type : "gauge",
|
||||||
|
help : "See following URL: https://lucene.apache.org/solr/guide/collections-api.html#clusterstatus",
|
||||||
|
label_names : [],
|
||||||
|
label_values : [],
|
||||||
|
value : $value
|
||||||
|
}
|
||||||
|
|
||||||
|
...
|
||||||
|
|
||||||
|
queries:
|
||||||
|
- query:
|
||||||
|
collection: collection1
|
||||||
|
path: /select
|
||||||
|
params:
|
||||||
|
- q: "*:*"
|
||||||
|
- start: 0
|
||||||
|
- rows: 0
|
||||||
|
- json.facet: |-
|
||||||
|
{
|
||||||
|
category: {
|
||||||
|
type: terms,
|
||||||
|
field: cat
|
||||||
|
}
|
||||||
|
}
|
||||||
|
jsonQueries:
|
||||||
|
# solr_facets_category
|
||||||
|
- |-
|
||||||
|
.facets.category.buckets[] as $object |
|
||||||
|
$object.val as $term |
|
||||||
|
$object.count as $value |
|
||||||
|
{
|
||||||
|
name : "solr_facets_category",
|
||||||
|
type : "gauge",
|
||||||
|
help : "Category facets",
|
||||||
|
label_names : ["term"],
|
||||||
|
label_values : [$term],
|
||||||
|
value : $value
|
||||||
|
}
|
||||||
|
----
|
||||||
|
|
||||||
|
|===
|
||||||
|
|Name|Description
|
||||||
|
|
||||||
|
|ping|Scrape <<ping.adoc#ping,Ping>> response.
|
||||||
|
|metrics|Scrape <<metrics-reporting.adoc#metrics-api,Metrics API>> response.
|
||||||
|
|collections|Scrape <<collections-api.adoc#collections-api,Collections API>> response.
|
||||||
|
|queries|Scrape <<searching.adoc#searching,Search API>> response.
|
||||||
|
|*.query|Query parameter for each features. You can specify `collection`, `core`, `path`, and `params`.
|
||||||
|
|*.jsonQueries|JSON Query that is jq syntax. For more details, see https://stedolan.github.io/jq/manual/[https://stedolan.github.io/jq/manual/].
|
||||||
|
|===
|
||||||
|
|
||||||
|
jq query has to output JSON in the following format.
|
||||||
|
|
||||||
|
[source,json]
|
||||||
|
----
|
||||||
|
{
|
||||||
|
name : "solr_ping",
|
||||||
|
type : "GAUGE",
|
||||||
|
help : "See following URL: https://lucene.apache.org/solr/guide/ping.html",
|
||||||
|
label_names : ["base_url","core"],
|
||||||
|
label_values : ["http://localhost:8983/solr","collection1"],
|
||||||
|
value : 1.0
|
||||||
|
}
|
||||||
|
----
|
||||||
|
|
||||||
|
It will be converted to the following exposition format.
|
||||||
|
|
||||||
|
[source,plain]
|
||||||
|
----
|
||||||
|
# TYPE solr_ping gauge
|
||||||
|
# HELP solr_ping See following URL: https://lucene.apache.org/solr/guide/ping.html
|
||||||
|
solr_ping{base_url="http://localhost:8983/solr",core="collection1"} 1.0
|
||||||
|
----
|
||||||
|
|
||||||
|
|===
|
||||||
|
|Name|Description
|
||||||
|
|
||||||
|
|name|The metric name to set. For more details, see https://prometheus.io/docs/practices/naming/[https://prometheus.io/docs/practices/naming/].
|
||||||
|
|type|The type of the metric, can be `COUNTER`, `GAUGE`, `SUMMARY`, `HISTOGRAM` or `UNTYPED`. For more detauils, see https://prometheus.io/docs/concepts/metric_types/[https://prometheus.io/docs/concepts/metric_types/].
|
||||||
|
|help|Help text for the metric.
|
||||||
|
|label_names|Label names for the metric. For more details, see https://prometheus.io/docs/practices/naming/[https://prometheus.io/docs/practices/naming/].
|
||||||
|
|label_values|Label values for the metric. For more details, see https://prometheus.io/docs/practices/naming/[https://prometheus.io/docs/practices/naming/].
|
||||||
|
|value|Value for the metric. Value must be set to Double type.
|
||||||
|
|===
|
||||||
|
|
||||||
|
|
||||||
|
== Prometheus Settings
|
||||||
|
|
||||||
|
You need to specify the solr-exporter listen address into `scrape_configs` in `prometheus.yml`. See following example:
|
||||||
|
|
||||||
|
[source,plain]
|
||||||
|
----
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: 'solr'
|
||||||
|
static_configs:
|
||||||
|
- targets: ['localhost:9983']
|
||||||
|
----
|
||||||
|
|
||||||
|
When you apply the above settings to prometheus, it will start to pull Solr's metrics from solr-exporter.
|
||||||
|
|
||||||
|
|
||||||
|
== Grafana Dashboard
|
||||||
|
|
||||||
|
A Grafana sample dashboard is provided at the following JSON file.
|
||||||
|
|
||||||
|
`./conf/grafana-solr-dashboard.json`
|
||||||
|
|
||||||
|
.Grafana Dashboard
|
||||||
|
image::images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png[image,width=800]
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
= Monitoring Solr
|
= Monitoring Solr
|
||||||
:page-children: metrics-reporting, mbean-request-handler, configuring-logging, using-jmx-with-solr, performance-statistics-reference
|
:page-children: metrics-reporting, mbean-request-handler, configuring-logging, using-jmx-with-solr, monitoring-solr-with-prometheus-and-grafana, performance-statistics-reference
|
||||||
// Licensed to the Apache Software Foundation (ASF) under one
|
// Licensed to the Apache Software Foundation (ASF) under one
|
||||||
// or more contributor license agreements. See the NOTICE file
|
// or more contributor license agreements. See the NOTICE file
|
||||||
// distributed with this work for additional information
|
// distributed with this work for additional information
|
||||||
|
|
@ -30,6 +30,8 @@ Common administrative tasks include:
|
||||||
|
|
||||||
<<using-jmx-with-solr.adoc#using-jmx-with-solr,Using JMX with Solr>>: Describes how to use Java Management Extensions with Solr.
|
<<using-jmx-with-solr.adoc#using-jmx-with-solr,Using JMX with Solr>>: Describes how to use Java Management Extensions with Solr.
|
||||||
|
|
||||||
|
<<monitoring-solr-with-prometheus-and-grafana.adoc#monitoring-solr-with-prometheus-and-grafana,Monitoring Solr with Prometheus and Grafana>>: Describes how to monitor Solr with Prometheus and Grafana.
|
||||||
|
|
||||||
<<performance-statistics-reference.adoc#performance-statistics-reference,Performance Statistics Reference>>: Additional information on statistics returned from JMX.
|
<<performance-statistics-reference.adoc#performance-statistics-reference,Performance Statistics Reference>>: Additional information on statistics returned from JMX.
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue