mirror of https://github.com/apache/lucene.git
SOLR-6671: Possible to set solr.data.home property as root dir for all data
This commit is contained in:
Jan Høydahl
parent
0ec9d64d81
commit
39dfb7808a
|
|
@ -909,6 +909,7 @@ New Features
|
|||
parsers' behavior remains the same: queries will be split on whitespace before sending individual terms to analysis.
|
||||
(Steve Rowe)
|
||||
|
||||
* SOLR-6671: Possible to set solr.data.home property as root dir for all data (janhoy, Shawn Heisey, Mark Miller)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
|
|
|||
|
|
@ -299,7 +299,7 @@ function print_usage() {
|
|||
echo ""
|
||||
elif [[ "$CMD" == "start" || "$CMD" == "restart" ]]; then
|
||||
echo ""
|
||||
echo "Usage: solr $CMD [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-a \"additional-options\"] [-V]"
|
||||
echo "Usage: solr $CMD [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-t solr.data.home] [-a \"additional-options\"] [-V]"
|
||||
echo ""
|
||||
echo " -f Start Solr in foreground; default starts Solr in the background"
|
||||
echo " and sends stdout / stderr to solr-PORT-console.log"
|
||||
|
|
@ -329,6 +329,9 @@ function print_usage() {
|
|||
echo " This parameter is ignored when running examples (-e), as the solr.solr.home depends"
|
||||
echo " on which example is run. The default value is server/solr."
|
||||
echo ""
|
||||
echo " -t <dir> Sets the solr.data.home system property, where Solr will store data (index)."
|
||||
echo " If not set, Solr uses solr.solr.home for config and data."
|
||||
echo ""
|
||||
echo " -e <example> Name of the example to run; available examples:"
|
||||
echo " cloud: SolrCloud example"
|
||||
echo " techproducts: Comprehensive example illustrating many of Solr's core capabilities"
|
||||
|
|
@ -1387,6 +1390,10 @@ if [ $# -gt 0 ]; then
|
|||
SOLR_HOME="$2"
|
||||
shift 2
|
||||
;;
|
||||
-t|-data.home)
|
||||
SOLR_DATA_HOME="$2"
|
||||
shift 2
|
||||
;;
|
||||
-e|-example)
|
||||
if [[ -z "$2" || "${2:0:1}" == "-" ]]; then
|
||||
print_usage "$SCRIPT_CMD" "Example name is required when using the $1 option!"
|
||||
|
|
@ -1682,6 +1689,10 @@ if [ ! -e "$SOLR_HOME" ]; then
|
|||
echo -e "\nSolr home directory $SOLR_HOME not found!\n"
|
||||
exit 1
|
||||
fi
|
||||
if [[ $SOLR_DATA_HOME ]] && [ ! -e "$SOLR_DATA_HOME" ]; then
|
||||
echo -e "\nSolr data home directory $SOLR_DATA_HOME not found!\n"
|
||||
exit 1
|
||||
fi
|
||||
if $verbose ; then
|
||||
q=""
|
||||
else
|
||||
|
|
@ -1882,6 +1893,9 @@ function launch_solr() {
|
|||
echo -e " SOLR_LOG_LEVEL = $SOLR_LOG_LEVEL"
|
||||
fi
|
||||
|
||||
if [ "$SOLR_DATA_HOME" != "" ]; then
|
||||
echo -e " SOLR_DATA_HOME = $SOLR_DATA_HOME"
|
||||
fi
|
||||
echo -e "\n"
|
||||
fi
|
||||
|
||||
|
|
@ -1897,7 +1911,7 @@ function launch_solr() {
|
|||
"${REMOTE_JMX_OPTS[@]}" "${CLOUD_MODE_OPTS[@]}" $SOLR_LOG_LEVEL_OPT -Dsolr.log.dir="$SOLR_LOGS_DIR" \
|
||||
"-Djetty.port=$SOLR_PORT" "-DSTOP.PORT=$stop_port" "-DSTOP.KEY=$STOP_KEY" \
|
||||
"${SOLR_HOST_ARG[@]}" "-Duser.timezone=$SOLR_TIMEZONE" \
|
||||
"-Djetty.home=$SOLR_SERVER_DIR" "-Dsolr.solr.home=$SOLR_HOME" "-Dsolr.install.dir=$SOLR_TIP" \
|
||||
"-Djetty.home=$SOLR_SERVER_DIR" "-Dsolr.solr.home=$SOLR_HOME" "-Dsolr.data.home=$SOLR_DATA_HOME" "-Dsolr.install.dir=$SOLR_TIP" \
|
||||
"${LOG4J_CONFIG[@]}" "${SOLR_OPTS[@]}")
|
||||
|
||||
if [ "$SOLR_MODE" == "solrcloud" ]; then
|
||||
|
|
|
|||
|
|
@ -285,7 +285,7 @@ goto done
|
|||
|
||||
:start_usage
|
||||
@echo.
|
||||
@echo Usage: solr %SCRIPT_CMD% [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-a "additional-options"] [-V]
|
||||
@echo Usage: solr %SCRIPT_CMD% [-f] [-c] [-h hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-t solr.data.home] [-a "additional-options"] [-V]
|
||||
@echo.
|
||||
@echo -f Start Solr in foreground; default starts Solr in the background
|
||||
@echo and sends stdout / stderr to solr-PORT-console.log
|
||||
|
|
@ -312,6 +312,9 @@ goto done
|
|||
@echo This parameter is ignored when running examples (-e), as the solr.solr.home depends
|
||||
@echo on which example is run. The default value is server/solr.
|
||||
@echo.
|
||||
@echo -t dir Sets the solr.data.home system property, used as root for ^<instance_dir^>/data directories.
|
||||
@echo If not set, Solr uses solr.solr.home for both config and data.
|
||||
@echo.
|
||||
@echo -e example Name of the example to run; available examples:
|
||||
@echo cloud: SolrCloud example
|
||||
@echo techproducts: Comprehensive example illustrating many of Solr's core capabilities
|
||||
|
|
@ -602,6 +605,7 @@ IF "%1"=="-cloud" goto set_cloud_mode
|
|||
IF "%1"=="-d" goto set_server_dir
|
||||
IF "%1"=="-dir" goto set_server_dir
|
||||
IF "%1"=="-s" goto set_solr_home_dir
|
||||
IF "%1"=="-t" goto set_solr_data_dir
|
||||
IF "%1"=="-solr.home" goto set_solr_home_dir
|
||||
IF "%1"=="-e" goto set_example
|
||||
IF "%1"=="-example" goto set_example
|
||||
|
|
@ -697,6 +701,24 @@ SHIFT
|
|||
SHIFT
|
||||
goto parse_args
|
||||
|
||||
:set_solr_data_dir
|
||||
|
||||
set "arg=%~2"
|
||||
IF "%arg%"=="" (
|
||||
set SCRIPT_ERROR=Directory name is required!
|
||||
goto invalid_cmd_line
|
||||
)
|
||||
|
||||
set firstChar=%arg:~0,1%
|
||||
IF "%firstChar%"=="-" (
|
||||
set SCRIPT_ERROR=Expected directory but found %2 instead!
|
||||
goto invalid_cmd_line
|
||||
)
|
||||
set "SOLR_DATA_HOME=%~2"
|
||||
SHIFT
|
||||
SHIFT
|
||||
goto parse_args
|
||||
|
||||
:set_example
|
||||
|
||||
set "arg=%~2"
|
||||
|
|
@ -897,6 +919,7 @@ IF [%SOLR_LOGS_DIR%] == [] (
|
|||
set SOLR_LOGS_DIR=%SOLR_LOGS_DIR:"=%
|
||||
)
|
||||
set SOLR_LOGS_DIR_QUOTED="%SOLR_LOGS_DIR%"
|
||||
set SOLR_DATA_HOME_QUOTED="%SOLR_DATA_HOME%"
|
||||
|
||||
set "EXAMPLE_DIR=%SOLR_TIP%\example"
|
||||
set TMP=!SOLR_HOME:%EXAMPLE_DIR%=!
|
||||
|
|
@ -1139,6 +1162,10 @@ IF "%verbose%"=="1" (
|
|||
@echo SOLR_LOG_LEVEL = !SOLR_LOG_LEVEL!
|
||||
)
|
||||
|
||||
IF NOT "%SOLR_DATA_HOME%"=="" (
|
||||
@echo SOLR_DATA_HOME = !SOLR_DATA_HOME!
|
||||
)
|
||||
|
||||
@echo.
|
||||
)
|
||||
|
||||
|
|
@ -1155,6 +1182,7 @@ IF "%SOLR_SSL_ENABLED%"=="true" (
|
|||
)
|
||||
IF NOT "%SOLR_LOG_LEVEL%"=="" set "START_OPTS=%START_OPTS% -Dsolr.log.level=%SOLR_LOG_LEVEL%"
|
||||
set "START_OPTS=%START_OPTS% -Dsolr.log.dir=%SOLR_LOGS_DIR_QUOTED%"
|
||||
IF NOT "%SOLR_DATA_HOME%"=="" set "START_OPTS=%START_OPTS% -Dsolr.data.home=%SOLR_DATA_HOME_QUOTED%"
|
||||
IF NOT DEFINED LOG4J_CONFIG set "LOG4J_CONFIG=file:%SOLR_SERVER_DIR%\resources\log4j.properties"
|
||||
|
||||
cd /d "%SOLR_SERVER_DIR%"
|
||||
|
|
@ -1833,4 +1861,4 @@ REM Safe echo which does not mess with () in strings
|
|||
set "eout=%1"
|
||||
set eout=%eout:"=%
|
||||
echo !eout!
|
||||
GOTO :eof
|
||||
GOTO :eof
|
||||
|
|
@ -68,6 +68,10 @@ REM Path to a directory for Solr to store cores and their data. By default, Solr
|
|||
REM If solr.xml is not stored in ZooKeeper, this directory needs to contain solr.xml
|
||||
REM set SOLR_HOME=
|
||||
|
||||
REM Path to a directory that Solr will use as root for data folders for each core.
|
||||
REM If not set, defaults to <instance_dir>/data. Overridable per core through 'dataDir' core property
|
||||
REM set SOLR_DATA_HOME=
|
||||
|
||||
REM Changes the logging level. Valid values: ALL, TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF. Default is INFO
|
||||
REM This is an alternative to changing the rootLogger in log4j.properties
|
||||
REM set SOLR_LOG_LEVEL=INFO
|
||||
|
|
|
|||
|
|
@ -86,6 +86,10 @@
|
|||
# If solr.xml is not stored in ZooKeeper, this directory needs to contain solr.xml
|
||||
#SOLR_HOME=
|
||||
|
||||
# Path to a directory that Solr will use as root for data folders for each core.
|
||||
# If not set, defaults to <instance_dir>/data. Overridable per core through 'dataDir' core property
|
||||
#SOLR_DATA_HOME=
|
||||
|
||||
# Solr provides a default Log4J configuration properties file in server/resources
|
||||
# however, you may want to customize the log settings and file appender location
|
||||
# so you can point the script to use a different log4j.properties file
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package org.apache.solr.core;
|
|||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
|
|
@ -395,6 +396,15 @@ public abstract class CachingDirectoryFactory extends DirectoryFactory {
|
|||
maxWriteMBPerSecMerge = (Double) args.get("maxWriteMBPerSecMerge");
|
||||
maxWriteMBPerSecRead = (Double) args.get("maxWriteMBPerSecRead");
|
||||
maxWriteMBPerSecDefault = (Double) args.get("maxWriteMBPerSecDefault");
|
||||
|
||||
dataHomePath = args.get(DATA_HOME) == null ? null : Paths.get((String) args.get(DATA_HOME));
|
||||
if (dataHomePath == null && System.getProperty(DATA_HOME) != null && System.getProperty(DATA_HOME).length() > 0) {
|
||||
// If solr.data.home is not specified explicitly in solrconfig.xml, fall back to global System Property with same name
|
||||
dataHomePath = Paths.get(System.getProperty(DATA_HOME));
|
||||
}
|
||||
if (dataHomePath != null) {
|
||||
log.info(DATA_HOME + "=" + dataHomePath);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ import java.io.IOException;
|
|||
import java.lang.invoke.MethodHandles;
|
||||
import java.nio.file.NoSuchFileException;
|
||||
import java.util.Arrays;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
|
|
@ -53,6 +55,11 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
|
|||
|
||||
protected static final String INDEX_W_TIMESTAMP_REGEX = "index\\.[0-9]{17}"; // see SnapShooter.DATE_FMT
|
||||
|
||||
public static final String DATA_HOME = "solr.data.home";
|
||||
|
||||
// May be set by sub classes as data root, in which case getDataHome will use it as base
|
||||
protected Path dataHomePath;
|
||||
|
||||
// hint about what the directory contains - default is index directory
|
||||
public enum DirContext {DEFAULT, META_DATA}
|
||||
|
||||
|
|
@ -315,9 +322,22 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
|
|||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the data home folder. If solr.data.home is set, that is used, else base on instanceDir
|
||||
* @param cd core descriptor instance
|
||||
* @return a String with absolute path to data direcotry
|
||||
*/
|
||||
public String getDataHome(CoreDescriptor cd) throws IOException {
|
||||
// by default, we go off the instance directory
|
||||
return cd.getInstanceDir().resolve(cd.getDataDir()).toAbsolutePath().toString();
|
||||
String dataDir;
|
||||
if (dataHomePath != null) {
|
||||
String instanceDirLastPath = cd.getInstanceDir().getName(cd.getInstanceDir().getNameCount()-1).toString();
|
||||
dataDir = Paths.get(coreContainer.getSolrHome()).resolve(dataHomePath)
|
||||
.resolve(instanceDirLastPath).resolve(cd.getDataDir()).toAbsolutePath().toString();
|
||||
} else {
|
||||
// by default, we go off the instance directory
|
||||
dataDir = cd.getInstanceDir().resolve(cd.getDataDir()).toAbsolutePath().toString();
|
||||
}
|
||||
return dataDir;
|
||||
}
|
||||
|
||||
public void cleanupOldIndexDirectories(final String dataDirPath, final String currentIndexDirPath, boolean afterCoreReload) {
|
||||
|
|
|
|||
|
|
@ -16,7 +16,17 @@
|
|||
*/
|
||||
package org.apache.solr.core;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.handler.admin.CoreAdminHandler;
|
||||
import org.apache.solr.handler.component.HttpShardHandlerFactory;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
public class DirectoryFactoryTest extends LuceneTestCase {
|
||||
|
||||
|
|
@ -28,4 +38,55 @@ public class DirectoryFactoryTest extends LuceneTestCase {
|
|||
assertEquals("hdfs", DirectoryFactory.LOCK_TYPE_HDFS);
|
||||
}
|
||||
|
||||
@After
|
||||
@Before
|
||||
public void clean() {
|
||||
System.clearProperty("solr.data.home");
|
||||
System.clearProperty("solr.solr.home");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetDataHome() throws Exception {
|
||||
MockCoreContainer cc = new MockCoreContainer("/solr/home");
|
||||
Properties cp = cc.getContainerProperties();
|
||||
boolean zkAware = cc.isZooKeeperAware();
|
||||
RAMDirectoryFactory rdf = new RAMDirectoryFactory();
|
||||
rdf.initCoreContainer(cc);
|
||||
rdf.init(new NamedList());
|
||||
|
||||
// No solr.data.home property set. Absolute instanceDir
|
||||
assertEquals("/tmp/inst1/data", rdf.getDataHome(new CoreDescriptor("core_name", Paths.get("/tmp/inst1"), cp, zkAware)));
|
||||
|
||||
// Simulate solr.data.home set in solrconfig.xml <directoryFactory> tag
|
||||
NamedList args = new NamedList();
|
||||
args.add("solr.data.home", "/solrdata/");
|
||||
rdf.init(args);
|
||||
assertEquals("/solrdata/inst_dir/data", rdf.getDataHome(new CoreDescriptor("core_name", Paths.get("inst_dir"), cp, zkAware)));
|
||||
|
||||
// solr.data.home set with System property, and relative path
|
||||
System.setProperty("solr.data.home", "solrdata");
|
||||
rdf.init(new NamedList());
|
||||
assertEquals("/solr/home/solrdata/inst_dir/data", rdf.getDataHome(new CoreDescriptor("core_name", Paths.get("inst_dir"), cp, zkAware)));
|
||||
// Test parsing last component of instanceDir, and using custom dataDir
|
||||
assertEquals("/solr/home/solrdata/myinst/mydata", rdf.getDataHome(new CoreDescriptor("core_name", Paths.get("/path/to/myinst"), cp, zkAware, "dataDir", "mydata")));
|
||||
}
|
||||
|
||||
|
||||
private static class MockCoreContainer extends CoreContainer {
|
||||
|
||||
private final String mockSolrHome;
|
||||
|
||||
public MockCoreContainer(String solrHome) throws IOException {
|
||||
super(new Object());
|
||||
mockSolrHome = solrHome;
|
||||
this.shardHandlerFactory = new HttpShardHandlerFactory();
|
||||
this.coreAdminHandler = new CoreAdminHandler();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSolrHome() {
|
||||
return mockSolrHome;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ The `${solr.core.name}` substitution will cause the name of the current core to
|
|||
|
||||
If you are using replication to replicate the Solr index (as described in <<legacy-scaling-and-distribution.adoc#legacy-scaling-and-distribution,Legacy Scaling and Distribution>>), then the `<dataDir>` directory should correspond to the index directory used in the replication configuration.
|
||||
|
||||
NOTE: If the environment variable `SOLR_DATA_HOME` if defined, or if `solr.data.home` is configured for your DirectoryFactory, the location of data directory will be `<SOLR_DATA_HOME>/<instance_name>/data`.
|
||||
|
||||
[[DataDirandDirectoryFactoryinSolrConfig-SpecifyingtheDirectoryFactoryForYourIndex]]
|
||||
== Specifying the DirectoryFactory For Your Index
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue