SOLR-6671: Possible to set solr.data.home property as root dir for all data

This commit is contained in:
Jan Høydahl 2017-06-20 13:20:44 +02:00
parent 0ec9d64d81
commit 39dfb7808a
9 changed files with 150 additions and 6 deletions

View File

@ -909,6 +909,7 @@ New Features
parsers' behavior remains the same: queries will be split on whitespace before sending individual terms to analysis.
(Steve Rowe)
* SOLR-6671: Possible to set solr.data.home property as root dir for all data (janhoy, Shawn Heisey, Mark Miller)
Bug Fixes
----------------------

View File

@ -299,7 +299,7 @@ function print_usage() {
echo ""
elif [[ "$CMD" == "start" || "$CMD" == "restart" ]]; then
echo ""
echo "Usage: solr $CMD [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-a \"additional-options\"] [-V]"
echo "Usage: solr $CMD [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-t solr.data.home] [-a \"additional-options\"] [-V]"
echo ""
echo " -f Start Solr in foreground; default starts Solr in the background"
echo " and sends stdout / stderr to solr-PORT-console.log"
@ -329,6 +329,9 @@ function print_usage() {
echo " This parameter is ignored when running examples (-e), as the solr.solr.home depends"
echo " on which example is run. The default value is server/solr."
echo ""
echo " -t <dir> Sets the solr.data.home system property, where Solr will store data (index)."
echo " If not set, Solr uses solr.solr.home for config and data."
echo ""
echo " -e <example> Name of the example to run; available examples:"
echo " cloud: SolrCloud example"
echo " techproducts: Comprehensive example illustrating many of Solr's core capabilities"
@ -1387,6 +1390,10 @@ if [ $# -gt 0 ]; then
SOLR_HOME="$2"
shift 2
;;
-t|-data.home)
SOLR_DATA_HOME="$2"
shift 2
;;
-e|-example)
if [[ -z "$2" || "${2:0:1}" == "-" ]]; then
print_usage "$SCRIPT_CMD" "Example name is required when using the $1 option!"
@ -1682,6 +1689,10 @@ if [ ! -e "$SOLR_HOME" ]; then
echo -e "\nSolr home directory $SOLR_HOME not found!\n"
exit 1
fi
if [[ $SOLR_DATA_HOME ]] && [ ! -e "$SOLR_DATA_HOME" ]; then
echo -e "\nSolr data home directory $SOLR_DATA_HOME not found!\n"
exit 1
fi
if $verbose ; then
q=""
else
@ -1882,6 +1893,9 @@ function launch_solr() {
echo -e " SOLR_LOG_LEVEL = $SOLR_LOG_LEVEL"
fi
if [ "$SOLR_DATA_HOME" != "" ]; then
echo -e " SOLR_DATA_HOME = $SOLR_DATA_HOME"
fi
echo -e "\n"
fi
@ -1897,7 +1911,7 @@ function launch_solr() {
"${REMOTE_JMX_OPTS[@]}" "${CLOUD_MODE_OPTS[@]}" $SOLR_LOG_LEVEL_OPT -Dsolr.log.dir="$SOLR_LOGS_DIR" \
"-Djetty.port=$SOLR_PORT" "-DSTOP.PORT=$stop_port" "-DSTOP.KEY=$STOP_KEY" \
"${SOLR_HOST_ARG[@]}" "-Duser.timezone=$SOLR_TIMEZONE" \
"-Djetty.home=$SOLR_SERVER_DIR" "-Dsolr.solr.home=$SOLR_HOME" "-Dsolr.install.dir=$SOLR_TIP" \
"-Djetty.home=$SOLR_SERVER_DIR" "-Dsolr.solr.home=$SOLR_HOME" "-Dsolr.data.home=$SOLR_DATA_HOME" "-Dsolr.install.dir=$SOLR_TIP" \
"${LOG4J_CONFIG[@]}" "${SOLR_OPTS[@]}")
if [ "$SOLR_MODE" == "solrcloud" ]; then

View File

@ -285,7 +285,7 @@ goto done
:start_usage
@echo.
@echo Usage: solr %SCRIPT_CMD% [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-a "additional-options"] [-V]
@echo Usage: solr %SCRIPT_CMD% [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-t solr.data.home] [-a "additional-options"] [-V]
@echo.
@echo -f Start Solr in foreground; default starts Solr in the background
@echo and sends stdout / stderr to solr-PORT-console.log
@ -312,6 +312,9 @@ goto done
@echo This parameter is ignored when running examples (-e), as the solr.solr.home depends
@echo on which example is run. The default value is server/solr.
@echo.
@echo -t dir Sets the solr.data.home system property, used as root for ^<instance_dir^>/data directories.
@echo If not set, Solr uses solr.solr.home for both config and data.
@echo.
@echo -e example Name of the example to run; available examples:
@echo cloud: SolrCloud example
@echo techproducts: Comprehensive example illustrating many of Solr's core capabilities
@ -602,6 +605,7 @@ IF "%1"=="-cloud" goto set_cloud_mode
IF "%1"=="-d" goto set_server_dir
IF "%1"=="-dir" goto set_server_dir
IF "%1"=="-s" goto set_solr_home_dir
IF "%1"=="-t" goto set_solr_data_dir
IF "%1"=="-solr.home" goto set_solr_home_dir
IF "%1"=="-e" goto set_example
IF "%1"=="-example" goto set_example
@ -697,6 +701,24 @@ SHIFT
SHIFT
goto parse_args
:set_solr_data_dir
set "arg=%~2"
IF "%arg%"=="" (
set SCRIPT_ERROR=Directory name is required!
goto invalid_cmd_line
)
set firstChar=%arg:~0,1%
IF "%firstChar%"=="-" (
set SCRIPT_ERROR=Expected directory but found %2 instead!
goto invalid_cmd_line
)
set "SOLR_DATA_HOME=%~2"
SHIFT
SHIFT
goto parse_args
:set_example
set "arg=%~2"
@ -897,6 +919,7 @@ IF [%SOLR_LOGS_DIR%] == [] (
set SOLR_LOGS_DIR=%SOLR_LOGS_DIR:"=%
)
set SOLR_LOGS_DIR_QUOTED="%SOLR_LOGS_DIR%"
set SOLR_DATA_HOME_QUOTED="%SOLR_DATA_HOME%"
set "EXAMPLE_DIR=%SOLR_TIP%\example"
set TMP=!SOLR_HOME:%EXAMPLE_DIR%=!
@ -1139,6 +1162,10 @@ IF "%verbose%"=="1" (
@echo SOLR_LOG_LEVEL = !SOLR_LOG_LEVEL!
)
IF NOT "%SOLR_DATA_HOME%"=="" (
@echo SOLR_DATA_HOME = !SOLR_DATA_HOME!
)
@echo.
)
@ -1155,6 +1182,7 @@ IF "%SOLR_SSL_ENABLED%"=="true" (
)
IF NOT "%SOLR_LOG_LEVEL%"=="" set "START_OPTS=%START_OPTS% -Dsolr.log.level=%SOLR_LOG_LEVEL%"
set "START_OPTS=%START_OPTS% -Dsolr.log.dir=%SOLR_LOGS_DIR_QUOTED%"
IF NOT "%SOLR_DATA_HOME%"=="" set "START_OPTS=%START_OPTS% -Dsolr.data.home=%SOLR_DATA_HOME_QUOTED%"
IF NOT DEFINED LOG4J_CONFIG set "LOG4J_CONFIG=file:%SOLR_SERVER_DIR%\resources\log4j.properties"
cd /d "%SOLR_SERVER_DIR%"
@ -1833,4 +1861,4 @@ REM Safe echo which does not mess with () in strings
set "eout=%1"
set eout=%eout:"=%
echo !eout!
GOTO :eof
GOTO :eof

View File

@ -68,6 +68,10 @@ REM Path to a directory for Solr to store cores and their data. By default, Solr
REM If solr.xml is not stored in ZooKeeper, this directory needs to contain solr.xml
REM set SOLR_HOME=
REM Path to a directory that Solr will use as root for data folders for each core.
REM If not set, defaults to <instance_dir>/data. Overridable per core through 'dataDir' core property
REM set SOLR_DATA_HOME=
REM Changes the logging level. Valid values: ALL, TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF. Default is INFO
REM This is an alternative to changing the rootLogger in log4j.properties
REM set SOLR_LOG_LEVEL=INFO

View File

@ -86,6 +86,10 @@
# If solr.xml is not stored in ZooKeeper, this directory needs to contain solr.xml
#SOLR_HOME=
# Path to a directory that Solr will use as root for data folders for each core.
# If not set, defaults to <instance_dir>/data. Overridable per core through 'dataDir' core property
#SOLR_DATA_HOME=
# Solr provides a default Log4J configuration properties file in server/resources
# however, you may want to customize the log settings and file appender location
# so you can point the script to use a different log4j.properties file

View File

@ -19,6 +19,7 @@ package org.apache.solr.core;
import java.io.File;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@ -395,6 +396,15 @@ public abstract class CachingDirectoryFactory extends DirectoryFactory {
maxWriteMBPerSecMerge = (Double) args.get("maxWriteMBPerSecMerge");
maxWriteMBPerSecRead = (Double) args.get("maxWriteMBPerSecRead");
maxWriteMBPerSecDefault = (Double) args.get("maxWriteMBPerSecDefault");
dataHomePath = args.get(DATA_HOME) == null ? null : Paths.get((String) args.get(DATA_HOME));
if (dataHomePath == null && System.getProperty(DATA_HOME) != null && System.getProperty(DATA_HOME).length() > 0) {
// If solr.data.home is not specified explicitly in solrconfig.xml, fall back to global System Property with same name
dataHomePath = Paths.get(System.getProperty(DATA_HOME));
}
if (dataHomePath != null) {
log.info(DATA_HOME + "=" + dataHomePath);
}
}
/*

View File

@ -24,6 +24,8 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.NoSuchFileException;
import java.util.Arrays;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collections;
import java.util.List;
@ -53,6 +55,11 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
protected static final String INDEX_W_TIMESTAMP_REGEX = "index\\.[0-9]{17}"; // see SnapShooter.DATE_FMT
public static final String DATA_HOME = "solr.data.home";
// May be set by sub classes as data root, in which case getDataHome will use it as base
protected Path dataHomePath;
// hint about what the directory contains - default is index directory
public enum DirContext {DEFAULT, META_DATA}
@ -315,9 +322,22 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
return false;
}
/**
* Get the data home folder. If solr.data.home is set, that is used, else base on instanceDir
* @param cd core descriptor instance
* @return a String with absolute path to data direcotry
*/
public String getDataHome(CoreDescriptor cd) throws IOException {
// by default, we go off the instance directory
return cd.getInstanceDir().resolve(cd.getDataDir()).toAbsolutePath().toString();
String dataDir;
if (dataHomePath != null) {
String instanceDirLastPath = cd.getInstanceDir().getName(cd.getInstanceDir().getNameCount()-1).toString();
dataDir = Paths.get(coreContainer.getSolrHome()).resolve(dataHomePath)
.resolve(instanceDirLastPath).resolve(cd.getDataDir()).toAbsolutePath().toString();
} else {
// by default, we go off the instance directory
dataDir = cd.getInstanceDir().resolve(cd.getDataDir()).toAbsolutePath().toString();
}
return dataDir;
}
public void cleanupOldIndexDirectories(final String dataDirPath, final String currentIndexDirPath, boolean afterCoreReload) {

View File

@ -16,7 +16,17 @@
*/
package org.apache.solr.core;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.Properties;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class DirectoryFactoryTest extends LuceneTestCase {
@ -28,4 +38,55 @@ public class DirectoryFactoryTest extends LuceneTestCase {
assertEquals("hdfs", DirectoryFactory.LOCK_TYPE_HDFS);
}
@After
@Before
public void clean() {
System.clearProperty("solr.data.home");
System.clearProperty("solr.solr.home");
}
@Test
public void testGetDataHome() throws Exception {
MockCoreContainer cc = new MockCoreContainer("/solr/home");
Properties cp = cc.getContainerProperties();
boolean zkAware = cc.isZooKeeperAware();
RAMDirectoryFactory rdf = new RAMDirectoryFactory();
rdf.initCoreContainer(cc);
rdf.init(new NamedList());
// No solr.data.home property set. Absolute instanceDir
assertEquals("/tmp/inst1/data", rdf.getDataHome(new CoreDescriptor("core_name", Paths.get("/tmp/inst1"), cp, zkAware)));
// Simulate solr.data.home set in solrconfig.xml <directoryFactory> tag
NamedList args = new NamedList();
args.add("solr.data.home", "/solrdata/");
rdf.init(args);
assertEquals("/solrdata/inst_dir/data", rdf.getDataHome(new CoreDescriptor("core_name", Paths.get("inst_dir"), cp, zkAware)));
// solr.data.home set with System property, and relative path
System.setProperty("solr.data.home", "solrdata");
rdf.init(new NamedList());
assertEquals("/solr/home/solrdata/inst_dir/data", rdf.getDataHome(new CoreDescriptor("core_name", Paths.get("inst_dir"), cp, zkAware)));
// Test parsing last component of instanceDir, and using custom dataDir
assertEquals("/solr/home/solrdata/myinst/mydata", rdf.getDataHome(new CoreDescriptor("core_name", Paths.get("/path/to/myinst"), cp, zkAware, "dataDir", "mydata")));
}
private static class MockCoreContainer extends CoreContainer {
private final String mockSolrHome;
public MockCoreContainer(String solrHome) throws IOException {
super(new Object());
mockSolrHome = solrHome;
this.shardHandlerFactory = new HttpShardHandlerFactory();
this.coreAdminHandler = new CoreAdminHandler();
}
@Override
public String getSolrHome() {
return mockSolrHome;
}
}
}

View File

@ -33,6 +33,8 @@ The `${solr.core.name}` substitution will cause the name of the current core to
If you are using replication to replicate the Solr index (as described in <<legacy-scaling-and-distribution.adoc#legacy-scaling-and-distribution,Legacy Scaling and Distribution>>), then the `<dataDir>` directory should correspond to the index directory used in the replication configuration.
NOTE: If the environment variable `SOLR_DATA_HOME` if defined, or if `solr.data.home` is configured for your DirectoryFactory, the location of data directory will be `<SOLR_DATA_HOME>/<instance_name>/data`.
[[DataDirandDirectoryFactoryinSolrConfig-SpecifyingtheDirectoryFactoryForYourIndex]]
== Specifying the DirectoryFactory For Your Index