LUCENE-5650: Enforce read-only access to any path outside the temporary folder via security manager

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1596767 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Dawid Weiss 2014-05-22 06:15:26 +00:00
parent 8ca3abe369
commit cf5a0caeb6
38 changed files with 438 additions and 295 deletions

View File

@ -273,6 +273,10 @@ Bug fixes
* LUCENE-5671: Upgrade ICU version to fix an ICU concurrency problem that * LUCENE-5671: Upgrade ICU version to fix an ICU concurrency problem that
could cause exceptions when indexing. (feedly team, Robert Muir) could cause exceptions when indexing. (feedly team, Robert Muir)
* LUCENE-5650: Enforce read-only access to any path outside the temporary
folder via security manager, and make test temp dirs absolute.
(Ryan Ernst, Dawid Weiss)
======================= Lucene 4.8.0 ======================= ======================= Lucene 4.8.0 =======================
System Requirements System Requirements

View File

@ -974,9 +974,9 @@
<!-- TODO: create propertyset for test properties, so each project can have its own set --> <!-- TODO: create propertyset for test properties, so each project can have its own set -->
<sysproperty key="tests.multiplier" value="@{tests.multiplier}"/> <sysproperty key="tests.multiplier" value="@{tests.multiplier}"/>
<!-- Temporary directory in the cwd. --> <!-- Temporary directory a subdir of the cwd. -->
<sysproperty key="tempDir" value="." /> <sysproperty key="tempDir" value="./temp" />
<sysproperty key="java.io.tmpdir" value="." /> <sysproperty key="java.io.tmpdir" value="./temp" />
<!-- Restrict access to certain Java features and install security manager: --> <!-- Restrict access to certain Java features and install security manager: -->
<sysproperty key="junit4.tempDir" file="@{workDir}/temp" /> <sysproperty key="junit4.tempDir" file="@{workDir}/temp" />

View File

@ -240,7 +240,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
searcherMgr = new SearcherManager(writer, true, null); searcherMgr = new SearcherManager(writer, true, null);
success = true; success = true;
} finally { } finally {
if (success == false) { if (success == false && writer != null) {
writer.rollback(); writer.rollback();
writer = null; writer = null;
} }

View File

@ -158,7 +158,6 @@ import com.carrotsearch.randomizedtesting.rules.NoClassHooksShadowingRule;
import com.carrotsearch.randomizedtesting.rules.NoInstanceHooksOverridesRule; import com.carrotsearch.randomizedtesting.rules.NoInstanceHooksOverridesRule;
import com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule; import com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule;
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule; import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule;
import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean; import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean;
import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsInt; import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsInt;
@ -498,6 +497,11 @@ public abstract class LuceneTestCase extends Assert {
*/ */
private static TestRuleMarkFailure suiteFailureMarker; private static TestRuleMarkFailure suiteFailureMarker;
/**
* Temporary files cleanup rule.
*/
private static TestRuleTemporaryFilesCleanup tempFilesCleanupRule;
/** /**
* Ignore tests after hitting a designated number of initial failures. This * Ignore tests after hitting a designated number of initial failures. This
* is truly a "static" global singleton since it needs to span the lifetime of all * is truly a "static" global singleton since it needs to span the lifetime of all
@ -573,7 +577,7 @@ public abstract class LuceneTestCase extends Assert {
.around(suiteFailureMarker = new TestRuleMarkFailure()) .around(suiteFailureMarker = new TestRuleMarkFailure())
.around(new TestRuleAssertionsRequired()) .around(new TestRuleAssertionsRequired())
.around(new TestRuleLimitSysouts(suiteFailureMarker)) .around(new TestRuleLimitSysouts(suiteFailureMarker))
.around(new TemporaryFilesCleanupRule()) .around(tempFilesCleanupRule = new TestRuleTemporaryFilesCleanup(suiteFailureMarker))
.around(new StaticFieldsInvariantRule(STATIC_LEAK_THRESHOLD, true) { .around(new StaticFieldsInvariantRule(STATIC_LEAK_THRESHOLD, true) {
@Override @Override
protected boolean accept(java.lang.reflect.Field field) { protected boolean accept(java.lang.reflect.Field field) {
@ -2364,53 +2368,13 @@ public abstract class LuceneTestCase extends Assert {
} }
} }
/**
* A base location for temporary files of a given test. Helps in figuring out
* which tests left which files and where.
*/
private static File tempDirBase;
/**
* Retry to create temporary file name this many times.
*/
private static final int TEMP_NAME_RETRY_THRESHOLD = 9999;
/** /**
* This method is deprecated for a reason. Do not use it. Call {@link #createTempDir()} * This method is deprecated for a reason. Do not use it. Call {@link #createTempDir()}
* or {@link #createTempDir(String)} or {@link #createTempFile(String, String)}. * or {@link #createTempDir(String)} or {@link #createTempFile(String, String)}.
*/ */
@Deprecated @Deprecated
public static File getBaseTempDirForTestClass() { public static File getBaseTempDirForTestClass() {
synchronized (LuceneTestCase.class) { return tempFilesCleanupRule.getPerTestClassTempDir();
if (tempDirBase == null) {
File directory = new File(System.getProperty("tempDir", System.getProperty("java.io.tmpdir")));
assert directory.exists() &&
directory.isDirectory() &&
directory.canWrite();
RandomizedContext ctx = RandomizedContext.current();
Class<?> clazz = ctx.getTargetClass();
String prefix = clazz.getName();
prefix = prefix.replaceFirst("^org.apache.lucene.", "lucene.");
prefix = prefix.replaceFirst("^org.apache.solr.", "solr.");
int attempt = 0;
File f;
do {
if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) {
throw new RuntimeException(
"Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
+ directory.getAbsolutePath());
}
f = new File(directory, prefix + "-" + ctx.getRunnerSeedAsString()
+ "-" + String.format(Locale.ENGLISH, "%03d", attempt));
} while (!f.mkdirs());
tempDirBase = f;
registerToRemoveAfterSuite(tempDirBase);
}
}
return tempDirBase;
} }
@ -2432,21 +2396,7 @@ public abstract class LuceneTestCase extends Assert {
* the folder from being removed. * the folder from being removed.
*/ */
public static File createTempDir(String prefix) { public static File createTempDir(String prefix) {
File base = getBaseTempDirForTestClass(); return tempFilesCleanupRule.createTempDir(prefix);
int attempt = 0;
File f;
do {
if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) {
throw new RuntimeException(
"Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
+ base.getAbsolutePath());
}
f = new File(base, prefix + "-" + String.format(Locale.ENGLISH, "%03d", attempt));
} while (!f.mkdirs());
registerToRemoveAfterSuite(f);
return f;
} }
/** /**
@ -2458,21 +2408,7 @@ public abstract class LuceneTestCase extends Assert {
* the folder from being removed. * the folder from being removed.
*/ */
public static File createTempFile(String prefix, String suffix) throws IOException { public static File createTempFile(String prefix, String suffix) throws IOException {
File base = getBaseTempDirForTestClass(); return tempFilesCleanupRule.createTempFile(prefix, suffix);
int attempt = 0;
File f;
do {
if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) {
throw new RuntimeException(
"Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
+ base.getAbsolutePath());
}
f = new File(base, prefix + "-" + String.format(Locale.ENGLISH, "%03d", attempt) + suffix);
} while (!f.createNewFile());
registerToRemoveAfterSuite(f);
return f;
} }
/** /**
@ -2483,79 +2419,4 @@ public abstract class LuceneTestCase extends Assert {
public static File createTempFile() throws IOException { public static File createTempFile() throws IOException {
return createTempFile("tempFile", ".tmp"); return createTempFile("tempFile", ".tmp");
} }
/**
* A queue of temporary resources to be removed after the
* suite completes.
* @see #registerToRemoveAfterSuite(File)
*/
private final static List<File> cleanupQueue = new ArrayList<File>();
/**
* Register temporary folder for removal after the suite completes.
*/
private static void registerToRemoveAfterSuite(File f) {
assert f != null;
if (LuceneTestCase.LEAVE_TEMPORARY) {
System.err.println("INFO: Will leave temporary file: " + f.getAbsolutePath());
return;
}
synchronized (cleanupQueue) {
cleanupQueue.add(f);
}
}
/**
* Checks and cleans up temporary files.
*
* @see LuceneTestCase#createTempDir()
* @see LuceneTestCase#createTempFile()
*/
private static class TemporaryFilesCleanupRule extends TestRuleAdapter {
@Override
protected void before() throws Throwable {
super.before();
assert tempDirBase == null;
}
@Override
protected void afterAlways(List<Throwable> errors) throws Throwable {
// Drain cleanup queue and clear it.
final File [] everything;
final String tempDirBasePath;
synchronized (cleanupQueue) {
tempDirBasePath = (tempDirBase != null ? tempDirBase.getAbsolutePath() : null);
tempDirBase = null;
Collections.reverse(cleanupQueue);
everything = new File [cleanupQueue.size()];
cleanupQueue.toArray(everything);
cleanupQueue.clear();
}
// Only check and throw an IOException on un-removable files if the test
// was successful. Otherwise just report the path of temporary files
// and leave them there.
if (LuceneTestCase.suiteFailureMarker.wasSuccessful()) {
try {
TestUtil.rm(everything);
} catch (IOException e) {
Class<?> suiteClass = RandomizedContext.current().getTargetClass();
if (suiteClass.isAnnotationPresent(SuppressTempFileChecks.class)) {
System.err.println("WARNING: Leftover undeleted temporary files (bugUrl: "
+ suiteClass.getAnnotation(SuppressTempFileChecks.class).bugUrl() + "): "
+ e.getMessage());
return;
}
throw e;
}
} else {
if (tempDirBasePath != null) {
System.err.println("NOTE: leaving temporary files on disk at: " + tempDirBasePath);
}
}
}
}
} }

View File

@ -0,0 +1,210 @@
package org.apache.lucene.util;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import org.apache.lucene.util.LuceneTestCase.SuppressTempFileChecks;
import com.carrotsearch.randomizedtesting.RandomizedContext;
import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Checks and cleans up temporary files.
*
* @see LuceneTestCase#createTempDir()
* @see LuceneTestCase#createTempFile()
*/
final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter {
/**
* Retry to create temporary file name this many times.
*/
private static final int TEMP_NAME_RETRY_THRESHOLD = 9999;
/**
* Writeable temporary base folder.
*/
private File javaTempDir;
/**
* Per-test class temporary folder.
*/
private File tempDirBase;
/**
* Suite failure marker.
*/
private final TestRuleMarkFailure failureMarker;
/**
* A queue of temporary resources to be removed after the
* suite completes.
* @see #registerToRemoveAfterSuite(File)
*/
private final static List<File> cleanupQueue = new ArrayList<File>();
public TestRuleTemporaryFilesCleanup(TestRuleMarkFailure failureMarker) {
this.failureMarker = failureMarker;
}
/**
* Register temporary folder for removal after the suite completes.
*/
void registerToRemoveAfterSuite(File f) {
assert f != null;
if (LuceneTestCase.LEAVE_TEMPORARY) {
System.err.println("INFO: Will leave temporary file: " + f.getAbsolutePath());
return;
}
synchronized (cleanupQueue) {
cleanupQueue.add(f);
}
}
@Override
protected void before() throws Throwable {
super.before();
assert tempDirBase == null;
javaTempDir = initializeJavaTempDir();
}
private File initializeJavaTempDir() {
File javaTempDir = new File(System.getProperty("tempDir", System.getProperty("java.io.tmpdir")));
if (!javaTempDir.exists() && !javaTempDir.mkdirs()) {
throw new RuntimeException("Could not create temp dir: " + javaTempDir.getAbsolutePath());
}
assert javaTempDir.isDirectory() &&
javaTempDir.canWrite();
return javaTempDir.getAbsoluteFile();
}
@Override
protected void afterAlways(List<Throwable> errors) throws Throwable {
// Drain cleanup queue and clear it.
final File [] everything;
final String tempDirBasePath;
synchronized (cleanupQueue) {
tempDirBasePath = (tempDirBase != null ? tempDirBase.getAbsolutePath() : null);
tempDirBase = null;
Collections.reverse(cleanupQueue);
everything = new File [cleanupQueue.size()];
cleanupQueue.toArray(everything);
cleanupQueue.clear();
}
// Only check and throw an IOException on un-removable files if the test
// was successful. Otherwise just report the path of temporary files
// and leave them there.
if (failureMarker.wasSuccessful()) {
try {
TestUtil.rm(everything);
} catch (IOException e) {
Class<?> suiteClass = RandomizedContext.current().getTargetClass();
if (suiteClass.isAnnotationPresent(SuppressTempFileChecks.class)) {
System.err.println("WARNING: Leftover undeleted temporary files (bugUrl: "
+ suiteClass.getAnnotation(SuppressTempFileChecks.class).bugUrl() + "): "
+ e.getMessage());
return;
}
throw e;
}
} else {
if (tempDirBasePath != null) {
System.err.println("NOTE: leaving temporary files on disk at: " + tempDirBasePath);
}
}
}
final File getPerTestClassTempDir() {
if (tempDirBase == null) {
RandomizedContext ctx = RandomizedContext.current();
Class<?> clazz = ctx.getTargetClass();
String prefix = clazz.getName();
prefix = prefix.replaceFirst("^org.apache.lucene.", "lucene.");
prefix = prefix.replaceFirst("^org.apache.solr.", "solr.");
int attempt = 0;
File f;
do {
if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) {
throw new RuntimeException(
"Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
+ javaTempDir.getAbsolutePath());
}
f = new File(javaTempDir, prefix + "-" + ctx.getRunnerSeedAsString()
+ "-" + String.format(Locale.ENGLISH, "%03d", attempt));
} while (!f.mkdirs());
tempDirBase = f;
registerToRemoveAfterSuite(tempDirBase);
}
return tempDirBase;
}
/**
* @see LuceneTestCase#createTempDir()
*/
public File createTempDir(String prefix) {
File base = getPerTestClassTempDir();
int attempt = 0;
File f;
do {
if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) {
throw new RuntimeException(
"Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
+ base.getAbsolutePath());
}
f = new File(base, prefix + "-" + String.format(Locale.ENGLISH, "%03d", attempt));
} while (!f.mkdirs());
registerToRemoveAfterSuite(f);
return f;
}
/**
* @see LuceneTestCase#createTempFile()
*/
public File createTempFile(String prefix, String suffix) throws IOException {
File base = getPerTestClassTempDir();
int attempt = 0;
File f;
do {
if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) {
throw new RuntimeException(
"Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
+ base.getAbsolutePath());
}
f = new File(base, prefix + "-" + String.format(Locale.ENGLISH, "%03d", attempt) + suffix);
} while (!f.createNewFile());
registerToRemoveAfterSuite(f);
return f;
}
}

View File

@ -26,8 +26,9 @@
grant { grant {
// permissions for file access, write access only to sandbox: // permissions for file access, write access only to sandbox:
permission java.io.FilePermission "<<ALL FILES>>", "read,execute"; permission java.io.FilePermission "<<ALL FILES>>", "read,execute";
permission java.io.FilePermission "${junit4.childvm.cwd}", "read,execute,write"; permission java.io.FilePermission "${junit4.childvm.cwd}", "read,execute";
permission java.io.FilePermission "${junit4.childvm.cwd}${/}-", "read,execute,write,delete"; permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp", "read,execute,write,delete";
permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp${/}-", "read,execute,write,delete";
permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,execute,write,delete"; permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,execute,write,delete";
permission java.io.FilePermission "${clover.db.dir}${/}-", "read,execute,write,delete"; permission java.io.FilePermission "${clover.db.dir}${/}-", "read,execute,write,delete";

View File

@ -151,6 +151,9 @@ Other Changes
* SOLR-5340: Add support for named snapshots (Varun Thacker via Noble Paul) * SOLR-5340: Add support for named snapshots (Varun Thacker via Noble Paul)
* LUCENE-5650: Tests can no longer write to CWD. Update log dir is now made relative
to the instance dir if it is not an absolute path. (Ryan Ernst, Dawid Weiss)
* SOLR-5681: Make the processing of Collection API calls multi-threaded. (Anshum Gupta) * SOLR-5681: Make the processing of Collection API calls multi-threaded. (Anshum Gupta)
* SOLR-5495: Recovery strategy for leader partitioned from replica case. Hardening * SOLR-5495: Recovery strategy for leader partitioned from replica case. Hardening

View File

@ -17,6 +17,9 @@ package org.apache.solr.analysis;
* limitations under the License. * limitations under the License.
*/ */
import java.io.File;
import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.SolrTestCaseJ4;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
@ -29,7 +32,9 @@ public class TestFoldingMultitermExtrasQuery extends SolrTestCaseJ4 {
@BeforeClass @BeforeClass
public static void beforeTests() throws Exception { public static void beforeTests() throws Exception {
initCore("solrconfig-icucollate.xml","schema-folding-extra.xml", "analysis-extras/solr"); File testHome = createTempDir();
FileUtils.copyDirectory(getFile("analysis-extras/solr"), testHome);
initCore("solrconfig-icucollate.xml","schema-folding-extra.xml", testHome.getAbsolutePath());
int idx = 1; int idx = 1;
// ICUFoldingFilterFactory // ICUFoldingFilterFactory

View File

@ -17,16 +17,21 @@ package org.apache.solr.schema;
* limitations under the License. * limitations under the License.
*/ */
import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.SolrTestCaseJ4;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import java.io.File;
/** /**
* Tests expert options of {@link ICUCollationField}. * Tests expert options of {@link ICUCollationField}.
*/ */
public class TestICUCollationFieldOptions extends SolrTestCaseJ4 { public class TestICUCollationFieldOptions extends SolrTestCaseJ4 {
@BeforeClass @BeforeClass
public static void beforeClass() throws Exception { public static void beforeClass() throws Exception {
initCore("solrconfig-icucollate.xml","schema-icucollateoptions.xml", "analysis-extras/solr"); File testHome = createTempDir();
FileUtils.copyDirectory(getFile("analysis-extras/solr"), testHome);
initCore("solrconfig-icucollate.xml","schema-icucollateoptions.xml", testHome.getAbsolutePath());
// add some docs // add some docs
assertU(adoc("id", "1", "text", "foo-bar")); assertU(adoc("id", "1", "text", "foo-bar"));
assertU(adoc("id", "2", "text", "foo bar")); assertU(adoc("id", "2", "text", "foo bar"));

View File

@ -16,8 +16,10 @@ package org.apache.solr.handler.clustering;
* limitations under the License. * limitations under the License.
*/ */
import java.io.File;
import java.util.Map; import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputDocument;
import org.junit.BeforeClass; import org.junit.BeforeClass;
@ -31,7 +33,9 @@ public abstract class AbstractClusteringTestCase extends SolrTestCaseJ4 {
@BeforeClass @BeforeClass
public static void beforeClass() throws Exception { public static void beforeClass() throws Exception {
initCore("solrconfig.xml", "schema.xml", "clustering/solr"); File testHome = createTempDir();
FileUtils.copyDirectory(getFile("clustering/solr"), testHome);
initCore("solrconfig.xml", "schema.xml", testHome.getAbsolutePath());
numberOfDocs = 0; numberOfDocs = 0;
for (String[] doc : DOCUMENTS) { for (String[] doc : DOCUMENTS) {
assertNull(h.validateUpdate(adoc("id", Integer.toString(numberOfDocs), "url", doc[0], "title", doc[1], "snippet", doc[2]))); assertNull(h.validateUpdate(adoc("id", Integer.toString(numberOfDocs), "url", doc[0], "title", doc[1], "snippet", doc[2])));

View File

@ -23,9 +23,6 @@
Data Import Handler Data Import Handler
</description> </description>
<!-- the tests have some parallel problems: writability to single copy of dataimport.properties -->
<property name="tests.jvms.override" value="1"/>
<import file="../contrib-build.xml"/> <import file="../contrib-build.xml"/>
<path id="test.classpath"> <path id="test.classpath">

View File

@ -27,6 +27,7 @@ import java.io.InputStreamReader;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.io.Writer; import java.io.Writer;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.security.AccessControlException;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;
@ -37,6 +38,7 @@ import java.util.Properties;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
/** /**
@ -113,22 +115,34 @@ public class SimplePropertiesWriter extends DIHProperties {
configDir = params.get(DIRECTORY); configDir = params.get(DIRECTORY);
} else { } else {
SolrCore core = dataImporter.getCore(); SolrCore core = dataImporter.getCore();
configDir = (core == null ? "." : core.getResourceLoader().getConfigDir()); if (core == null) {
configDir = SolrResourceLoader.locateSolrHome();
} else {
configDir = core.getResourceLoader().getConfigDir();
}
} }
} }
private File getPersistFile() { private File getPersistFile() {
String filePath = configDir; final File filePath;
if (configDir != null && !configDir.endsWith(File.separator)) filePath += File.separator; if (new File(filename).isAbsolute() || configDir == null) {
filePath += filename; filePath = new File(filename);
return new File(filePath); } else {
filePath = new File(new File(configDir), filename);
} }
return filePath;
}
@Override @Override
public boolean isWritable() { public boolean isWritable() {
File persistFile = getPersistFile(); File persistFile = getPersistFile();
return persistFile.exists() ? persistFile.canWrite() : persistFile try {
.getParentFile().canWrite(); return persistFile.exists()
? persistFile.canWrite()
: persistFile.getParentFile().canWrite();
} catch (AccessControlException e) {
return false;
}
} }
@Override @Override
@ -188,12 +202,7 @@ public class SimplePropertiesWriter extends DIHProperties {
Properties newProps = mapToProperties(propObjs); Properties newProps = mapToProperties(propObjs);
try { try {
existingProps.putAll(newProps); existingProps.putAll(newProps);
String filePath = configDir; propOutput = new OutputStreamWriter(new FileOutputStream(getPersistFile()), StandardCharsets.UTF_8);
if (configDir != null && !configDir.endsWith(File.separator)) {
filePath += File.separator;
}
filePath += filename;
propOutput = new OutputStreamWriter(new FileOutputStream(filePath), StandardCharsets.UTF_8);
existingProps.store(propOutput, null); existingProps.store(propOutput, null);
log.info("Wrote last indexed time to " + filename); log.info("Wrote last indexed time to " + filename);
} catch (Exception e) { } catch (Exception e) {

View File

@ -1,6 +1,5 @@
package org.apache.solr.handler.dataimport.config; package org.apache.solr.handler.dataimport.config;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -23,11 +22,11 @@ import java.util.Map;
public class PropertyWriter { public class PropertyWriter {
private final String type; private final String type;
private final Map<String,String> parameters; private Map<String,String> parameters;
public PropertyWriter(String type, Map<String,String> parameters) { public PropertyWriter(String type, Map<String,String> parameters) {
this.type = type; this.type = type;
this.parameters = Collections.unmodifiableMap(new HashMap<>(parameters)); this.parameters = new HashMap<String,String>(parameters);
} }
public Map<String,String> getParameters() { public Map<String,String> getParameters() {

View File

@ -16,7 +16,17 @@
*/ */
package org.apache.solr.handler.dataimport; package org.apache.solr.handler.dataimport;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
@ -28,18 +38,8 @@ import org.apache.solr.update.MergeIndexesCommand;
import org.apache.solr.update.RollbackUpdateCommand; import org.apache.solr.update.RollbackUpdateCommand;
import org.apache.solr.update.processor.UpdateRequestProcessor; import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory; import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
import org.apache.solr.common.util.NamedList;
import org.junit.After;
import org.junit.Before; import org.junit.Before;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.File;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/** /**
* <p> * <p>
* Abstract base class for DataImportHandler tests * Abstract base class for DataImportHandler tests
@ -55,27 +55,17 @@ public abstract class AbstractDataImportHandlerTestCase extends
// note, a little twisted that we shadow this static method // note, a little twisted that we shadow this static method
public static void initCore(String config, String schema) throws Exception { public static void initCore(String config, String schema) throws Exception {
initCore(config, schema, getFile("dih/solr").getAbsolutePath()); File testHome = createTempDir("core-home");
FileUtils.copyDirectory(getFile("dih/solr"), testHome);
initCore(config, schema, testHome.getAbsolutePath());
} }
@Override @Override
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
super.setUp(); super.setUp();
} File home = createTempDir("dih-properties");
System.setProperty("solr.solr.home", home.getAbsolutePath());
@Override
@After
public void tearDown() throws Exception {
// remove dataimport.properties
File f = new File("solr/collection1/conf/dataimport.properties");
log.info("Looking for dataimport.properties at: " + f.getAbsolutePath());
if (f.exists()) {
log.info("Deleting dataimport.properties");
if (!f.delete())
log.warn("Could not delete dataimport.properties");
}
super.tearDown();
} }
protected String loadDataConfig(String dataConfigFileName) { protected String loadDataConfig(String dataConfigFileName) {
@ -103,6 +93,21 @@ public abstract class AbstractDataImportHandlerTestCase extends
h.query("/dataimport", request); h.query("/dataimport", request);
} }
/**
* Redirect {@link SimplePropertiesWriter#filename} to a temporary location
* and return it.
*/
protected File redirectTempProperties(DataImporter di) {
try {
File tempFile = createTempFile();
di.getConfig().getPropertyWriter().getParameters()
.put(SimplePropertiesWriter.FILENAME, tempFile.getAbsolutePath());
return tempFile;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/** /**
* Runs a full-import using the given dataConfig and the provided request parameters. * Runs a full-import using the given dataConfig and the provided request parameters.
* *

View File

@ -73,6 +73,8 @@ public class TestDocBuilder extends AbstractDataImportHandlerTestCase {
public void testDeltaImportNoRows_MustNotCommit() { public void testDeltaImportNoRows_MustNotCommit() {
DataImporter di = new DataImporter(); DataImporter di = new DataImporter();
di.loadAndInit(dc_deltaConfig); di.loadAndInit(dc_deltaConfig);
redirectTempProperties(di);
DIHConfiguration cfg = di.getConfig(); DIHConfiguration cfg = di.getConfig();
Entity ent = cfg.getEntities().get(0); Entity ent = cfg.getEntities().get(0);
MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator()); MockDataSource.setIterator("select * from x", new ArrayList<Map<String, Object>>().iterator());

View File

@ -35,6 +35,8 @@ public class TestFieldReader extends AbstractDataImportHandlerTestCase {
public void simple() { public void simple() {
DataImporter di = new DataImporter(); DataImporter di = new DataImporter();
di.loadAndInit(config); di.loadAndInit(config);
redirectTempProperties(di);
TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl(); TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null); RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
List<Map<String, Object>> l = new ArrayList<>(); List<Map<String, Object>> l = new ArrayList<>();

View File

@ -72,8 +72,8 @@ public class TestNonWritablePersistFile extends AbstractDataImportHandlerTestCas
try { try {
// execute the test only if we are able to set file to read only mode // execute the test only if we are able to set file to read only mode
assumeTrue("No dataimport.properties file", f.exists() || f.createNewFile()); assumeTrue("No dataimport.properties file", f.exists() || f.createNewFile());
assumeTrue("dataimport.proprties can't be set read only", f.setReadOnly()); assumeTrue("dataimport.properties can't be set read only", f.setReadOnly());
assumeFalse("dataimport.proprties is still writable even though " + assumeFalse("dataimport.properties is still writable even though " +
"marked readonly - test running as superuser?", f.canWrite()); "marked readonly - test running as superuser?", f.canWrite());
ignoreException("Properties is not writable"); ignoreException("Properties is not writable");

View File

@ -16,11 +16,13 @@
*/ */
package org.apache.solr.handler.dataimport; package org.apache.solr.handler.dataimport;
import org.junit.Test; import java.io.File;
import java.io.IOException;
import java.io.StringReader; import java.io.StringReader;
import java.util.Properties; import java.util.Properties;
import org.junit.Test;
/** /**
* Test for PlainTextEntityProcessor * Test for PlainTextEntityProcessor
* *
@ -30,9 +32,11 @@ import java.util.Properties;
*/ */
public class TestPlainTextEntityProcessor extends AbstractDataImportHandlerTestCase { public class TestPlainTextEntityProcessor extends AbstractDataImportHandlerTestCase {
@Test @Test
public void testSimple() { public void testSimple() throws IOException {
DataImporter di = new DataImporter(); DataImporter di = new DataImporter();
di.loadAndInit(DATA_CONFIG); di.loadAndInit(DATA_CONFIG);
redirectTempProperties(di);
TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl(); TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl();
RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null); RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
di.runCmd(rp, sw); di.runCmd(rp, sw);

View File

@ -69,7 +69,7 @@ public class MorphlineBasicMiniMRTest extends SolrTestCaseJ4 {
private static final String DOCUMENTS_DIR = RESOURCES_DIR + "/test-documents"; private static final String DOCUMENTS_DIR = RESOURCES_DIR + "/test-documents";
private static final File MINIMR_CONF_DIR = new File(RESOURCES_DIR + "/solr/minimr"); private static final File MINIMR_CONF_DIR = new File(RESOURCES_DIR + "/solr/minimr");
private static final String SEARCH_ARCHIVES_JAR = JarFinder.getJar(MapReduceIndexerTool.class); private static String SEARCH_ARCHIVES_JAR;
private static MiniDFSCluster dfsCluster = null; private static MiniDFSCluster dfsCluster = null;
private static MiniMRCluster mrCluster = null; private static MiniMRCluster mrCluster = null;
@ -150,6 +150,9 @@ public class MorphlineBasicMiniMRTest extends SolrTestCaseJ4 {
System.setProperty("test.build.data", dataDir + File.separator + "hdfs" + File.separator + "build"); System.setProperty("test.build.data", dataDir + File.separator + "hdfs" + File.separator + "build");
System.setProperty("test.cache.data", dataDir + File.separator + "hdfs" + File.separator + "cache"); System.setProperty("test.cache.data", dataDir + File.separator + "hdfs" + File.separator + "cache");
// Initialize AFTER test.build.dir is set, JarFinder uses it.
SEARCH_ARCHIVES_JAR = JarFinder.getJar(MapReduceIndexerTool.class);
JobConf conf = new JobConf(); JobConf conf = new JobConf();
conf.set("dfs.block.access.token.enable", "false"); conf.set("dfs.block.access.token.enable", "false");
conf.set("dfs.permissions", "true"); conf.set("dfs.permissions", "true");

View File

@ -99,7 +99,7 @@ public class MorphlineGoLiveMiniMRTest extends AbstractFullDistribZkTestBase {
private static final File MINIMR_INSTANCE_DIR = new File(RESOURCES_DIR + "/solr/minimr"); private static final File MINIMR_INSTANCE_DIR = new File(RESOURCES_DIR + "/solr/minimr");
private static final File MINIMR_CONF_DIR = new File(RESOURCES_DIR + "/solr/minimr"); private static final File MINIMR_CONF_DIR = new File(RESOURCES_DIR + "/solr/minimr");
private static final String SEARCH_ARCHIVES_JAR = JarFinder.getJar(MapReduceIndexerTool.class); private static String SEARCH_ARCHIVES_JAR;
private static MiniDFSCluster dfsCluster = null; private static MiniDFSCluster dfsCluster = null;
private static MiniMRClientCluster mrCluster = null; private static MiniMRClientCluster mrCluster = null;
@ -173,6 +173,9 @@ public class MorphlineGoLiveMiniMRTest extends AbstractFullDistribZkTestBase {
System.setProperty("test.build.data", tempDir + File.separator + "hdfs" + File.separator + "build"); System.setProperty("test.build.data", tempDir + File.separator + "hdfs" + File.separator + "build");
System.setProperty("test.cache.data", tempDir + File.separator + "hdfs" + File.separator + "cache"); System.setProperty("test.cache.data", tempDir + File.separator + "hdfs" + File.separator + "cache");
// Initialize AFTER test.build.dir is set, JarFinder uses it.
SEARCH_ARCHIVES_JAR = JarFinder.getJar(MapReduceIndexerTool.class);
dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null); dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null);
FileSystem fileSystem = dfsCluster.getFileSystem(); FileSystem fileSystem = dfsCluster.getFileSystem();
fileSystem.mkdirs(new Path("/tmp")); fileSystem.mkdirs(new Path("/tmp"));

View File

@ -133,13 +133,16 @@ public class VelocityResponseWriter implements QueryResponseWriter {
// TODO: Externalize Velocity properties // TODO: Externalize Velocity properties
String propFile = request.getParams().get("v.properties"); String propFile = request.getParams().get("v.properties");
try { try {
if (propFile == null) Properties props = new Properties();
engine.init(); // Don't create a separate velocity log file by default.
else { props.put(RuntimeConstants.RUNTIME_LOG, "");
if (propFile == null) {
engine.init(props);
} else {
InputStream is = null; InputStream is = null;
try { try {
is = resourceLoader.getResourceStream(propFile); is = resourceLoader.getResourceStream(propFile);
Properties props = new Properties();
props.load(new InputStreamReader(is, StandardCharsets.UTF_8)); props.load(new InputStreamReader(is, StandardCharsets.UTF_8));
engine.init(props); engine.init(props);
} }

View File

@ -22,12 +22,10 @@ import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.response.VelocityResponseWriter; import org.apache.solr.response.VelocityResponseWriter;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import java.io.StringWriter; import java.io.StringWriter;
import java.io.IOException;
public class VelocityResponseWriterTest extends SolrTestCaseJ4 { public class VelocityResponseWriterTest extends SolrTestCaseJ4 {
@BeforeClass @BeforeClass

View File

@ -245,6 +245,7 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
public String getDataHome(CoreDescriptor cd) throws IOException { public String getDataHome(CoreDescriptor cd) throws IOException {
// by default, we go off the instance directory // by default, we go off the instance directory
return normalize(SolrResourceLoader.normalizeDir(cd.getInstanceDir()) + cd.getDataDir()); String instanceDir = new File(cd.getInstanceDir()).getAbsolutePath();
return normalize(SolrResourceLoader.normalizeDir(instanceDir) + cd.getDataDir());
} }
} }

View File

@ -161,6 +161,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
private final SolrResourceLoader resourceLoader; private final SolrResourceLoader resourceLoader;
private volatile IndexSchema schema; private volatile IndexSchema schema;
private final String dataDir; private final String dataDir;
private final String ulogDir;
private final UpdateHandler updateHandler; private final UpdateHandler updateHandler;
private final SolrCoreState solrCoreState; private final SolrCoreState solrCoreState;
@ -242,6 +243,10 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
return dataDir; return dataDir;
} }
public String getUlogDir() {
return ulogDir;
}
public String getIndexDir() { public String getIndexDir() {
synchronized (searcherLock) { synchronized (searcherLock) {
if (_searcher == null) return getNewIndexDir(); if (_searcher == null) return getNewIndexDir();
@ -654,6 +659,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
this.setName(name); this.setName(name);
this.schema = null; this.schema = null;
this.dataDir = null; this.dataDir = null;
this.ulogDir = null;
this.solrConfig = null; this.solrConfig = null;
this.startTime = System.currentTimeMillis(); this.startTime = System.currentTimeMillis();
this.maxWarmingSearchers = 2; // we don't have a config yet, just pick a number. this.maxWarmingSearchers = 2; // we don't have a config yet, just pick a number.
@ -701,8 +707,18 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
} }
} }
} }
dataDir = SolrResourceLoader.normalizeDir(dataDir); dataDir = SolrResourceLoader.normalizeDir(dataDir);
String updateLogDir = cd.getUlogDir();
if (updateLogDir == null) {
updateLogDir = dataDir;
if (new File(updateLogDir).isAbsolute() == false) {
updateLogDir = SolrResourceLoader.normalizeDir(cd.getInstanceDir()) + updateLogDir;
}
}
ulogDir = updateLogDir;
log.info(logid+"Opening new SolrCore at " + resourceLoader.getInstanceDir() + ", dataDir="+dataDir); log.info(logid+"Opening new SolrCore at " + resourceLoader.getInstanceDir() + ", dataDir="+dataDir);
if (null != cd && null != cd.getCloudDescriptor()) { if (null != cd && null != cd.getCloudDescriptor()) {

View File

@ -17,6 +17,7 @@ package org.apache.solr.spelling.suggest;
* limitations under the License. * limitations under the License.
*/ */
import java.io.Closeable;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
@ -28,6 +29,7 @@ import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.Lookup; import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CloseHook;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrIndexSearcher;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -101,6 +103,21 @@ public class SolrSuggester {
// initialize appropriate lookup instance // initialize appropriate lookup instance
factory = core.getResourceLoader().newInstance(lookupImpl, LookupFactory.class); factory = core.getResourceLoader().newInstance(lookupImpl, LookupFactory.class);
lookup = factory.create(config, core); lookup = factory.create(config, core);
core.addCloseHook(new CloseHook() {
@Override
public void preClose(SolrCore core) {
if (lookup != null && lookup instanceof Closeable) {
try {
((Closeable) lookup).close();
} catch (IOException e) {
LOG.warn("Could not close the suggester lookup.", e);
}
}
}
@Override
public void postClose(SolrCore core) {}
});
// if store directory is provided make it or load up the lookup with its content // if store directory is provided make it or load up the lookup with its content
if (store != null) { if (store != null) {

View File

@ -17,6 +17,7 @@
package org.apache.solr.spelling.suggest; package org.apache.solr.spelling.suggest;
import java.io.Closeable;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
@ -33,13 +34,14 @@ import org.apache.lucene.search.spell.Dictionary;
import org.apache.lucene.search.spell.HighFrequencyDictionary; import org.apache.lucene.search.spell.HighFrequencyDictionary;
import org.apache.lucene.search.spell.SuggestMode; import org.apache.lucene.search.spell.SuggestMode;
import org.apache.lucene.search.suggest.FileDictionary; import org.apache.lucene.search.suggest.FileDictionary;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.Lookup; import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester; import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester;
import org.apache.lucene.search.suggest.fst.WFSTCompletionLookup; import org.apache.lucene.search.suggest.fst.WFSTCompletionLookup;
import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CloseHook;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.spelling.SolrSpellChecker; import org.apache.solr.spelling.SolrSpellChecker;
@ -103,6 +105,22 @@ public class Suggester extends SolrSpellChecker {
factory = core.getResourceLoader().newInstance(lookupImpl, LookupFactory.class); factory = core.getResourceLoader().newInstance(lookupImpl, LookupFactory.class);
lookup = factory.create(config, core); lookup = factory.create(config, core);
core.addCloseHook(new CloseHook() {
@Override
public void preClose(SolrCore core) {
if (lookup != null && lookup instanceof Closeable) {
try {
((Closeable) lookup).close();
} catch (IOException e) {
LOG.warn("Could not close the suggester lookup.", e);
}
}
}
@Override
public void postClose(SolrCore core) {}
});
String store = (String)config.get(STORE_DIR); String store = (String)config.get(STORE_DIR);
if (store != null) { if (store != null) {
storeDir = new File(store); storeDir = new File(store);
@ -120,6 +138,7 @@ public class Suggester extends SolrSpellChecker {
} }
} }
} }
return name; return name;
} }

View File

@ -83,6 +83,9 @@ public class AnalyzingInfixLookupFactory extends LookupFactory {
String indexPath = params.get(INDEX_PATH) != null String indexPath = params.get(INDEX_PATH) != null
? params.get(INDEX_PATH).toString() ? params.get(INDEX_PATH).toString()
: DEFAULT_INDEX_PATH; : DEFAULT_INDEX_PATH;
if (new File(indexPath).isAbsolute() == false) {
indexPath = core.getDataDir() + File.separator + indexPath;
}
int minPrefixChars = params.get(MIN_PREFIX_CHARS) != null int minPrefixChars = params.get(MIN_PREFIX_CHARS) != null
? Integer.parseInt(params.get(MIN_PREFIX_CHARS).toString()) ? Integer.parseInt(params.get(MIN_PREFIX_CHARS).toString())

View File

@ -82,6 +82,9 @@ public class BlendedInfixLookupFactory extends AnalyzingInfixLookupFactory {
String indexPath = params.get(INDEX_PATH) != null String indexPath = params.get(INDEX_PATH) != null
? params.get(INDEX_PATH).toString() ? params.get(INDEX_PATH).toString()
: DEFAULT_INDEX_PATH; : DEFAULT_INDEX_PATH;
if (new File(indexPath).isAbsolute() == false) {
indexPath = core.getDataDir() + File.separator + indexPath;
}
int minPrefixChars = params.get(MIN_PREFIX_CHARS) != null int minPrefixChars = params.get(MIN_PREFIX_CHARS) != null
? Integer.parseInt(params.get(MIN_PREFIX_CHARS).toString()) ? Integer.parseInt(params.get(MIN_PREFIX_CHARS).toString())

View File

@ -222,15 +222,7 @@ public class UpdateLog implements PluginInfoInitialized {
* for an existing log whenever the core or update handler changes. * for an existing log whenever the core or update handler changes.
*/ */
public void init(UpdateHandler uhandler, SolrCore core) { public void init(UpdateHandler uhandler, SolrCore core) {
// ulogDir from CoreDescriptor overrides dataDir = core.getUlogDir();
String ulogDir = core.getCoreDescriptor().getUlogDir();
if (ulogDir != null) {
dataDir = ulogDir;
}
if (dataDir == null || dataDir.length()==0) {
dataDir = core.getDataDir();
}
this.uhandler = uhandler; this.uhandler = uhandler;

View File

@ -79,9 +79,8 @@ public class TestMiniSolrCloudCluster extends LuceneTestCase {
@BeforeClass @BeforeClass
public static void startup() throws Exception { public static void startup() throws Exception {
String testHome = SolrTestCaseJ4.TEST_HOME(); File solrXml = new File(SolrTestCaseJ4.TEST_HOME(), "solr-no-core.xml");
miniCluster = new MiniSolrCloudCluster(NUM_SERVERS, null, new File(testHome, "solr-no-core.xml"), miniCluster = new MiniSolrCloudCluster(NUM_SERVERS, null, createTempDir(), solrXml, null, null);
null, null);
} }
@AfterClass @AfterClass

View File

@ -56,17 +56,19 @@ public class TestConfigSets extends SolrTestCaseJ4 {
@Test @Test
public void testConfigSetServiceFindsConfigSets() { public void testConfigSetServiceFindsConfigSets() {
CoreContainer container = null; CoreContainer container = null;
SolrCore core1 = null;
try { try {
container = setupContainer(getFile("solr/configsets").getAbsolutePath()); container = setupContainer(getFile("solr/configsets").getAbsolutePath());
String testDirectory = container.getResourceLoader().getInstanceDir(); String testDirectory = new File(container.getResourceLoader().getInstanceDir()).getAbsolutePath();
SolrCore core1 = container.create("core1", testDirectory + "/core1", "configSet", "configset-2"); core1 = container.create("core1", testDirectory + "core1", "configSet", "configset-2");
assertThat(core1.getCoreDescriptor().getName(), is("core1")); assertThat(core1.getCoreDescriptor().getName(), is("core1"));
assertThat(core1.getDataDir(), is(testDirectory + "/core1" + File.separator + "data" + File.separator)); assertThat(core1.getDataDir(), is(testDirectory + "core1" + File.separator + "data" + File.separator));
core1.close();
} }
finally { finally {
if (core1 != null) {
core1.close();
}
if (container != null) if (container != null)
container.shutdown(); container.shutdown();
} }

View File

@ -114,12 +114,12 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
// System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory"); // System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
// For manual testing only // For manual testing only
// useFactory(null); // force an FS factory. // useFactory(null); // force an FS factory.
master = new SolrInstance("master", null); master = new SolrInstance(createTempDir("solr-instance"), "master", null);
master.setUp(); master.setUp();
masterJetty = createJetty(master); masterJetty = createJetty(master);
masterClient = createNewSolrServer(masterJetty.getLocalPort()); masterClient = createNewSolrServer(masterJetty.getLocalPort());
slave = new SolrInstance("slave", masterJetty.getLocalPort()); slave = new SolrInstance(createTempDir("solr-instance"), "slave", masterJetty.getLocalPort());
slave.setUp(); slave.setUp();
slaveJetty = createJetty(slave); slaveJetty = createJetty(slave);
slaveClient = createNewSolrServer(slaveJetty.getLocalPort()); slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
@ -323,7 +323,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
JettySolrRunner repeaterJetty = null; JettySolrRunner repeaterJetty = null;
SolrServer repeaterClient = null; SolrServer repeaterClient = null;
try { try {
repeater = new SolrInstance("repeater", masterJetty.getLocalPort()); repeater = new SolrInstance(createTempDir("solr-instance"), "repeater", masterJetty.getLocalPort());
repeater.setUp(); repeater.setUp();
repeaterJetty = createJetty(repeater); repeaterJetty = createJetty(repeater);
repeaterClient = createNewSolrServer(repeaterJetty.getLocalPort()); repeaterClient = createNewSolrServer(repeaterJetty.getLocalPort());
@ -911,7 +911,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
slaveClient = createNewSolrServer(slaveJetty.getLocalPort()); slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
try { try {
repeater = new SolrInstance("repeater", null); repeater = new SolrInstance(createTempDir("solr-instance"), "repeater", null);
repeater.setUp(); repeater.setUp();
repeater.copyConfigFile(CONF_DIR + "solrconfig-repeater.xml", repeater.copyConfigFile(CONF_DIR + "solrconfig-repeater.xml",
"solrconfig.xml"); "solrconfig.xml");
@ -1647,13 +1647,15 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
private File dataDir; private File dataDir;
/** /**
* @param name used to pick new solr home dir, as well as which * @param homeDir Base directory to build solr configuration and index in
* @param name used to pick which
* "solrconfig-${name}.xml" file gets copied * "solrconfig-${name}.xml" file gets copied
* to solrconfig.xml in new conf dir. * to solrconfig.xml in new conf dir.
* @param testPort if not null, used as a replacement for * @param testPort if not null, used as a replacement for
* TEST_PORT in the cloned config files. * TEST_PORT in the cloned config files.
*/ */
public SolrInstance(String name, Integer testPort) { public SolrInstance(File homeDir, String name, Integer testPort) {
this.homeDir = homeDir;
this.name = name; this.name = name;
this.testPort = testPort; this.testPort = testPort;
} }
@ -1687,11 +1689,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
System.setProperty("solr.test.sys.prop1", "propone"); System.setProperty("solr.test.sys.prop1", "propone");
System.setProperty("solr.test.sys.prop2", "proptwo"); System.setProperty("solr.test.sys.prop2", "proptwo");
File home = new File(dataDir,
getClass().getName() + "-" +
System.currentTimeMillis());
homeDir = new File(home, name);
dataDir = new File(homeDir + "/collection1", "data"); dataDir = new File(homeDir + "/collection1", "data");
confDir = new File(homeDir + "/collection1", "conf"); confDir = new File(homeDir + "/collection1", "conf");

View File

@ -16,6 +16,7 @@ package org.apache.solr.rest;
* limitations under the License. * limitations under the License.
*/ */
import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
@ -67,11 +68,11 @@ public class TestManagedResourceStorage extends AbstractZkTestCase {
*/ */
@Test @Test
public void testFileBasedJsonStorage() throws Exception { public void testFileBasedJsonStorage() throws Exception {
SolrResourceLoader loader = new SolrResourceLoader("./"); File instanceDir = createTempDir("json-storage");
// Solr unit tests can only write to their working directory due to SolrResourceLoader loader = new SolrResourceLoader(instanceDir.getAbsolutePath());
// a custom Java Security Manager installed in the test environment
NamedList<String> initArgs = new NamedList<>(); NamedList<String> initArgs = new NamedList<>();
initArgs.add(ManagedResourceStorage.STORAGE_DIR_INIT_ARG, "./managed"); String managedDir = instanceDir.getAbsolutePath() + File.separator + "managed";
initArgs.add(ManagedResourceStorage.STORAGE_DIR_INIT_ARG, managedDir);
FileStorageIO fileStorageIO = new FileStorageIO(); FileStorageIO fileStorageIO = new FileStorageIO();
fileStorageIO.configure(loader, initArgs); fileStorageIO.configure(loader, initArgs);
doStorageTests(loader, fileStorageIO); doStorageTests(loader, fileStorageIO);

View File

@ -1,10 +1,7 @@
package org.apache.solr.spelling.suggest; package org.apache.solr.spelling.suggest;
import java.io.File;
import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.SpellingParams; import org.apache.solr.common.params.SpellingParams;
import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
/* /*
@ -33,16 +30,6 @@ public class TestAnalyzeInfixSuggestions extends SolrTestCaseJ4 {
assertQ(req("qt", URI_DEFAULT, "q", "", SpellingParams.SPELLCHECK_BUILD, "true")); assertQ(req("qt", URI_DEFAULT, "q", "", SpellingParams.SPELLCHECK_BUILD, "true"));
} }
@AfterClass
public static void afterClass() throws Exception {
File indexPathDir = new File("analyzingInfixSuggesterIndexDir");
File indexPathDirTmp = new File("analyzingInfixSuggesterIndexDir.tmp");
if (indexPathDir.exists())
recurseDelete(indexPathDir);
if (indexPathDirTmp.exists())
recurseDelete(indexPathDirTmp);
}
public void testSingle() throws Exception { public void testSingle() throws Exception {
assertQ(req("qt", URI_DEFAULT, "q", "japan", SpellingParams.SPELLCHECK_COUNT, "1"), assertQ(req("qt", URI_DEFAULT, "q", "japan", SpellingParams.SPELLCHECK_COUNT, "1"),

View File

@ -17,10 +17,7 @@ package org.apache.solr.spelling.suggest;
* limitations under the License. * limitations under the License.
*/ */
import java.io.File;
import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.SolrTestCaseJ4;
import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
public class TestBlendedInfixSuggestions extends SolrTestCaseJ4 { public class TestBlendedInfixSuggestions extends SolrTestCaseJ4 {
@ -32,16 +29,6 @@ public class TestBlendedInfixSuggestions extends SolrTestCaseJ4 {
assertQ(req("qt", URI, "q", "", SuggesterParams.SUGGEST_BUILD_ALL, "true")); assertQ(req("qt", URI, "q", "", SuggesterParams.SUGGEST_BUILD_ALL, "true"));
} }
@AfterClass
public static void afterClass() throws Exception {
File indexPathDir = new File("blendedInfixSuggesterIndexDir");
File indexPathDirTmp = new File("blendedInfixSuggesterIndexDir.tmp");
if (indexPathDir.exists())
recurseDelete(indexPathDir);
if (indexPathDirTmp.exists())
recurseDelete(indexPathDirTmp);
}
public void testLinearBlenderType() { public void testLinearBlenderType() {
assertQ(req("qt", URI, "q", "the", SuggesterParams.SUGGEST_COUNT, "10", SuggesterParams.SUGGEST_DICT, "blended_infix_suggest_linear"), assertQ(req("qt", URI, "q", "the", SuggesterParams.SUGGEST_COUNT, "10", SuggesterParams.SUGGEST_DICT, "blended_infix_suggest_linear"),
"//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/int[@name='numFound'][.='3']", "//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/int[@name='numFound'][.='3']",
@ -55,7 +42,6 @@ public class TestBlendedInfixSuggestions extends SolrTestCaseJ4 {
"//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/arr[@name='suggestions']/lst[3]/long[@name='weight'][.='7']", "//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/arr[@name='suggestions']/lst[3]/long[@name='weight'][.='7']",
"//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='payload'][.='star']" "//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='payload'][.='star']"
); );
} }
public void testReciprocalBlenderType() { public void testReciprocalBlenderType() {
@ -97,5 +83,4 @@ public class TestBlendedInfixSuggestions extends SolrTestCaseJ4 {
"//lst[@name='suggest']/lst[@name='blended_infix_suggest_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='payload'][.='star']" "//lst[@name='suggest']/lst[@name='blended_infix_suggest_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='payload'][.='star']"
); );
} }
} }

View File

@ -92,7 +92,7 @@ public class TestLBHttpSolrServer extends SolrTestCaseJ4 {
httpClient = HttpClientUtil.createClient(null); httpClient = HttpClientUtil.createClient(null);
HttpClientUtil.setConnectionTimeout(httpClient, 1000); HttpClientUtil.setConnectionTimeout(httpClient, 1000);
for (int i = 0; i < solr.length; i++) { for (int i = 0; i < solr.length; i++) {
solr[i] = new SolrInstance("solr/collection1" + i, 0); solr[i] = new SolrInstance("solr/collection1" + i, createTempDir("instance-" + i), 0);
solr[i].setUp(); solr[i].setUp();
solr[i].startJetty(); solr[i].startJetty();
addDocs(solr[i]); addDocs(solr[i]);
@ -250,9 +250,13 @@ public class TestLBHttpSolrServer extends SolrTestCaseJ4 {
int port; int port;
JettySolrRunner jetty; JettySolrRunner jetty;
public SolrInstance(String name, int port) { public SolrInstance(String name, File homeDir, int port) {
this.name = name; this.name = name;
this.homeDir = homeDir;
this.port = port; this.port = port;
dataDir = new File(homeDir + "/collection1", "data");
confDir = new File(homeDir + "/collection1", "conf");
} }
public String getHomeDir() { public String getHomeDir() {
@ -285,12 +289,6 @@ public class TestLBHttpSolrServer extends SolrTestCaseJ4 {
public void setUp() throws Exception { public void setUp() throws Exception {
File home = new File(dataDir,
getClass().getName() + "-" + System.currentTimeMillis());
homeDir = new File(home, name);
dataDir = new File(homeDir + "/collection1", "data");
confDir = new File(homeDir + "/collection1", "conf");
homeDir.mkdirs(); homeDir.mkdirs();
dataDir.mkdirs(); dataDir.mkdirs();
confDir.mkdirs(); confDir.mkdirs();
@ -301,7 +299,6 @@ public class TestLBHttpSolrServer extends SolrTestCaseJ4 {
FileUtils.copyFile(SolrTestCaseJ4.getFile(getSolrConfigFile()), f); FileUtils.copyFile(SolrTestCaseJ4.getFile(getSolrConfigFile()), f);
f = new File(confDir, "schema.xml"); f = new File(confDir, "schema.xml");
FileUtils.copyFile(SolrTestCaseJ4.getFile(getSchemaFile()), f); FileUtils.copyFile(SolrTestCaseJ4.getFile(getSchemaFile()), f);
} }
public void tearDown() throws Exception { public void tearDown() throws Exception {

View File

@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.request;
import junit.framework.Assert; import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrJettyTestBase; import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.client.solrj.response.SolrPingResponse; import org.apache.solr.client.solrj.response.SolrPingResponse;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
@ -27,6 +28,8 @@ import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import java.io.File;
/** /**
* Test SolrPing in Solrj * Test SolrPing in Solrj
*/ */
@ -34,7 +37,9 @@ public class SolrPingTest extends SolrJettyTestBase {
@BeforeClass @BeforeClass
public static void beforeClass() throws Exception { public static void beforeClass() throws Exception {
initCore("solrconfig.xml", "schema.xml", "solrj/solr", "collection1"); File testHome = createTempDir();
FileUtils.copyDirectory(getFile("solrj/solr"), testHome);
initCore("solrconfig.xml", "schema.xml", testHome.getAbsolutePath(), "collection1");
} }
@Before @Before

View File

@ -48,14 +48,15 @@ public class MiniSolrCloudCluster {
* "Mini" SolrCloud cluster to be used for testing * "Mini" SolrCloud cluster to be used for testing
* @param numServers number of Solr servers to start * @param numServers number of Solr servers to start
* @param hostContext context path of Solr servers used by Jetty * @param hostContext context path of Solr servers used by Jetty
* @param baseDir base directory that the mini cluster should be run from
* @param solrXml solr.xml file to be uploaded to ZooKeeper * @param solrXml solr.xml file to be uploaded to ZooKeeper
* @param extraServlets Extra servlets to be started by Jetty * @param extraServlets Extra servlets to be started by Jetty
* @param extraRequestFilters extra filters to be started by Jetty * @param extraRequestFilters extra filters to be started by Jetty
*/ */
public MiniSolrCloudCluster(int numServers, String hostContext, File solrXml, public MiniSolrCloudCluster(int numServers, String hostContext, File baseDir, File solrXml,
SortedMap<ServletHolder, String> extraServlets, SortedMap<ServletHolder, String> extraServlets,
SortedMap<Class, String> extraRequestFilters) throws Exception { SortedMap<Class, String> extraRequestFilters) throws Exception {
testDir = Files.createTempDir(); testDir = baseDir;
String zkDir = testDir.getAbsolutePath() + File.separator String zkDir = testDir.getAbsolutePath() + File.separator
+ "zookeeper/server1/data"; + "zookeeper/server1/data";