diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index b8381e010cb..271263dd93a 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -273,6 +273,10 @@ Bug fixes * LUCENE-5671: Upgrade ICU version to fix an ICU concurrency problem that could cause exceptions when indexing. (feedly team, Robert Muir) +* LUCENE-5650: Enforce read-only access to any path outside the temporary + folder via security manager, and make test temp dirs absolute. + (Ryan Ernst, Dawid Weiss) + ======================= Lucene 4.8.0 ======================= System Requirements diff --git a/lucene/common-build.xml b/lucene/common-build.xml index eec9e5bdca2..3e5b3b06452 100644 --- a/lucene/common-build.xml +++ b/lucene/common-build.xml @@ -974,9 +974,9 @@ - - - + + + diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java index b454128f034..89e6b7874a9 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java @@ -240,7 +240,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable { searcherMgr = new SearcherManager(writer, true, null); success = true; } finally { - if (success == false) { + if (success == false && writer != null) { writer.rollback(); writer = null; } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java index adee188b5c9..f4f624a4b76 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java @@ -158,7 +158,6 @@ import com.carrotsearch.randomizedtesting.rules.NoClassHooksShadowingRule; import com.carrotsearch.randomizedtesting.rules.NoInstanceHooksOverridesRule; import com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule; import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule; -import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter; import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean; import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsInt; @@ -497,6 +496,11 @@ public abstract class LuceneTestCase extends Assert { * Suite failure marker (any error in the test or suite scope). */ private static TestRuleMarkFailure suiteFailureMarker; + + /** + * Temporary files cleanup rule. + */ + private static TestRuleTemporaryFilesCleanup tempFilesCleanupRule; /** * Ignore tests after hitting a designated number of initial failures. This @@ -573,7 +577,7 @@ public abstract class LuceneTestCase extends Assert { .around(suiteFailureMarker = new TestRuleMarkFailure()) .around(new TestRuleAssertionsRequired()) .around(new TestRuleLimitSysouts(suiteFailureMarker)) - .around(new TemporaryFilesCleanupRule()) + .around(tempFilesCleanupRule = new TestRuleTemporaryFilesCleanup(suiteFailureMarker)) .around(new StaticFieldsInvariantRule(STATIC_LEAK_THRESHOLD, true) { @Override protected boolean accept(java.lang.reflect.Field field) { @@ -2364,53 +2368,13 @@ public abstract class LuceneTestCase extends Assert { } } - /** - * A base location for temporary files of a given test. Helps in figuring out - * which tests left which files and where. - */ - private static File tempDirBase; - - /** - * Retry to create temporary file name this many times. - */ - private static final int TEMP_NAME_RETRY_THRESHOLD = 9999; - /** * This method is deprecated for a reason. Do not use it. Call {@link #createTempDir()} * or {@link #createTempDir(String)} or {@link #createTempFile(String, String)}. */ @Deprecated public static File getBaseTempDirForTestClass() { - synchronized (LuceneTestCase.class) { - if (tempDirBase == null) { - File directory = new File(System.getProperty("tempDir", System.getProperty("java.io.tmpdir"))); - assert directory.exists() && - directory.isDirectory() && - directory.canWrite(); - - RandomizedContext ctx = RandomizedContext.current(); - Class clazz = ctx.getTargetClass(); - String prefix = clazz.getName(); - prefix = prefix.replaceFirst("^org.apache.lucene.", "lucene."); - prefix = prefix.replaceFirst("^org.apache.solr.", "solr."); - - int attempt = 0; - File f; - do { - if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) { - throw new RuntimeException( - "Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: " - + directory.getAbsolutePath()); - } - f = new File(directory, prefix + "-" + ctx.getRunnerSeedAsString() - + "-" + String.format(Locale.ENGLISH, "%03d", attempt)); - } while (!f.mkdirs()); - - tempDirBase = f; - registerToRemoveAfterSuite(tempDirBase); - } - } - return tempDirBase; + return tempFilesCleanupRule.getPerTestClassTempDir(); } @@ -2432,21 +2396,7 @@ public abstract class LuceneTestCase extends Assert { * the folder from being removed. */ public static File createTempDir(String prefix) { - File base = getBaseTempDirForTestClass(); - - int attempt = 0; - File f; - do { - if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) { - throw new RuntimeException( - "Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: " - + base.getAbsolutePath()); - } - f = new File(base, prefix + "-" + String.format(Locale.ENGLISH, "%03d", attempt)); - } while (!f.mkdirs()); - - registerToRemoveAfterSuite(f); - return f; + return tempFilesCleanupRule.createTempDir(prefix); } /** @@ -2458,21 +2408,7 @@ public abstract class LuceneTestCase extends Assert { * the folder from being removed. */ public static File createTempFile(String prefix, String suffix) throws IOException { - File base = getBaseTempDirForTestClass(); - - int attempt = 0; - File f; - do { - if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) { - throw new RuntimeException( - "Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: " - + base.getAbsolutePath()); - } - f = new File(base, prefix + "-" + String.format(Locale.ENGLISH, "%03d", attempt) + suffix); - } while (!f.createNewFile()); - - registerToRemoveAfterSuite(f); - return f; + return tempFilesCleanupRule.createTempFile(prefix, suffix); } /** @@ -2483,79 +2419,4 @@ public abstract class LuceneTestCase extends Assert { public static File createTempFile() throws IOException { return createTempFile("tempFile", ".tmp"); } - - /** - * A queue of temporary resources to be removed after the - * suite completes. - * @see #registerToRemoveAfterSuite(File) - */ - private final static List cleanupQueue = new ArrayList(); - - /** - * Register temporary folder for removal after the suite completes. - */ - private static void registerToRemoveAfterSuite(File f) { - assert f != null; - - if (LuceneTestCase.LEAVE_TEMPORARY) { - System.err.println("INFO: Will leave temporary file: " + f.getAbsolutePath()); - return; - } - - synchronized (cleanupQueue) { - cleanupQueue.add(f); - } - } - - /** - * Checks and cleans up temporary files. - * - * @see LuceneTestCase#createTempDir() - * @see LuceneTestCase#createTempFile() - */ - private static class TemporaryFilesCleanupRule extends TestRuleAdapter { - @Override - protected void before() throws Throwable { - super.before(); - assert tempDirBase == null; - } - - @Override - protected void afterAlways(List errors) throws Throwable { - // Drain cleanup queue and clear it. - final File [] everything; - final String tempDirBasePath; - synchronized (cleanupQueue) { - tempDirBasePath = (tempDirBase != null ? tempDirBase.getAbsolutePath() : null); - tempDirBase = null; - - Collections.reverse(cleanupQueue); - everything = new File [cleanupQueue.size()]; - cleanupQueue.toArray(everything); - cleanupQueue.clear(); - } - - // Only check and throw an IOException on un-removable files if the test - // was successful. Otherwise just report the path of temporary files - // and leave them there. - if (LuceneTestCase.suiteFailureMarker.wasSuccessful()) { - try { - TestUtil.rm(everything); - } catch (IOException e) { - Class suiteClass = RandomizedContext.current().getTargetClass(); - if (suiteClass.isAnnotationPresent(SuppressTempFileChecks.class)) { - System.err.println("WARNING: Leftover undeleted temporary files (bugUrl: " - + suiteClass.getAnnotation(SuppressTempFileChecks.class).bugUrl() + "): " - + e.getMessage()); - return; - } - throw e; - } - } else { - if (tempDirBasePath != null) { - System.err.println("NOTE: leaving temporary files on disk at: " + tempDirBasePath); - } - } - } - } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java new file mode 100644 index 00000000000..52433e28afe --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java @@ -0,0 +1,210 @@ +package org.apache.lucene.util; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +import org.apache.lucene.util.LuceneTestCase.SuppressTempFileChecks; + +import com.carrotsearch.randomizedtesting.RandomizedContext; +import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Checks and cleans up temporary files. + * + * @see LuceneTestCase#createTempDir() + * @see LuceneTestCase#createTempFile() + */ +final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter { + /** + * Retry to create temporary file name this many times. + */ + private static final int TEMP_NAME_RETRY_THRESHOLD = 9999; + + /** + * Writeable temporary base folder. + */ + private File javaTempDir; + + /** + * Per-test class temporary folder. + */ + private File tempDirBase; + + /** + * Suite failure marker. + */ + private final TestRuleMarkFailure failureMarker; + + /** + * A queue of temporary resources to be removed after the + * suite completes. + * @see #registerToRemoveAfterSuite(File) + */ + private final static List cleanupQueue = new ArrayList(); + + public TestRuleTemporaryFilesCleanup(TestRuleMarkFailure failureMarker) { + this.failureMarker = failureMarker; + } + + /** + * Register temporary folder for removal after the suite completes. + */ + void registerToRemoveAfterSuite(File f) { + assert f != null; + + if (LuceneTestCase.LEAVE_TEMPORARY) { + System.err.println("INFO: Will leave temporary file: " + f.getAbsolutePath()); + return; + } + + synchronized (cleanupQueue) { + cleanupQueue.add(f); + } + } + + @Override + protected void before() throws Throwable { + super.before(); + + assert tempDirBase == null; + javaTempDir = initializeJavaTempDir(); + } + + private File initializeJavaTempDir() { + File javaTempDir = new File(System.getProperty("tempDir", System.getProperty("java.io.tmpdir"))); + if (!javaTempDir.exists() && !javaTempDir.mkdirs()) { + throw new RuntimeException("Could not create temp dir: " + javaTempDir.getAbsolutePath()); + } + assert javaTempDir.isDirectory() && + javaTempDir.canWrite(); + + return javaTempDir.getAbsoluteFile(); + } + + @Override + protected void afterAlways(List errors) throws Throwable { + // Drain cleanup queue and clear it. + final File [] everything; + final String tempDirBasePath; + synchronized (cleanupQueue) { + tempDirBasePath = (tempDirBase != null ? tempDirBase.getAbsolutePath() : null); + tempDirBase = null; + + Collections.reverse(cleanupQueue); + everything = new File [cleanupQueue.size()]; + cleanupQueue.toArray(everything); + cleanupQueue.clear(); + } + + // Only check and throw an IOException on un-removable files if the test + // was successful. Otherwise just report the path of temporary files + // and leave them there. + if (failureMarker.wasSuccessful()) { + try { + TestUtil.rm(everything); + } catch (IOException e) { + Class suiteClass = RandomizedContext.current().getTargetClass(); + if (suiteClass.isAnnotationPresent(SuppressTempFileChecks.class)) { + System.err.println("WARNING: Leftover undeleted temporary files (bugUrl: " + + suiteClass.getAnnotation(SuppressTempFileChecks.class).bugUrl() + "): " + + e.getMessage()); + return; + } + throw e; + } + } else { + if (tempDirBasePath != null) { + System.err.println("NOTE: leaving temporary files on disk at: " + tempDirBasePath); + } + } + } + + final File getPerTestClassTempDir() { + if (tempDirBase == null) { + RandomizedContext ctx = RandomizedContext.current(); + Class clazz = ctx.getTargetClass(); + String prefix = clazz.getName(); + prefix = prefix.replaceFirst("^org.apache.lucene.", "lucene."); + prefix = prefix.replaceFirst("^org.apache.solr.", "solr."); + + int attempt = 0; + File f; + do { + if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) { + throw new RuntimeException( + "Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: " + + javaTempDir.getAbsolutePath()); + } + f = new File(javaTempDir, prefix + "-" + ctx.getRunnerSeedAsString() + + "-" + String.format(Locale.ENGLISH, "%03d", attempt)); + } while (!f.mkdirs()); + + tempDirBase = f; + registerToRemoveAfterSuite(tempDirBase); + } + return tempDirBase; + } + + /** + * @see LuceneTestCase#createTempDir() + */ + public File createTempDir(String prefix) { + File base = getPerTestClassTempDir(); + + int attempt = 0; + File f; + do { + if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) { + throw new RuntimeException( + "Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: " + + base.getAbsolutePath()); + } + f = new File(base, prefix + "-" + String.format(Locale.ENGLISH, "%03d", attempt)); + } while (!f.mkdirs()); + + registerToRemoveAfterSuite(f); + return f; + } + + /** + * @see LuceneTestCase#createTempFile() + */ + public File createTempFile(String prefix, String suffix) throws IOException { + File base = getPerTestClassTempDir(); + + int attempt = 0; + File f; + do { + if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) { + throw new RuntimeException( + "Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: " + + base.getAbsolutePath()); + } + f = new File(base, prefix + "-" + String.format(Locale.ENGLISH, "%03d", attempt) + suffix); + } while (!f.createNewFile()); + + registerToRemoveAfterSuite(f); + return f; + } +} diff --git a/lucene/tools/junit4/tests.policy b/lucene/tools/junit4/tests.policy index b1c4311e3b0..98e3f2b33fa 100644 --- a/lucene/tools/junit4/tests.policy +++ b/lucene/tools/junit4/tests.policy @@ -26,8 +26,9 @@ grant { // permissions for file access, write access only to sandbox: permission java.io.FilePermission "<>", "read,execute"; - permission java.io.FilePermission "${junit4.childvm.cwd}", "read,execute,write"; - permission java.io.FilePermission "${junit4.childvm.cwd}${/}-", "read,execute,write,delete"; + permission java.io.FilePermission "${junit4.childvm.cwd}", "read,execute"; + permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp", "read,execute,write,delete"; + permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp${/}-", "read,execute,write,delete"; permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,execute,write,delete"; permission java.io.FilePermission "${clover.db.dir}${/}-", "read,execute,write,delete"; diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d8e91efa5cc..5f3d5cee074 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -151,6 +151,9 @@ Other Changes * SOLR-5340: Add support for named snapshots (Varun Thacker via Noble Paul) +* LUCENE-5650: Tests can no longer write to CWD. Update log dir is now made relative + to the instance dir if it is not an absolute path. (Ryan Ernst, Dawid Weiss) + * SOLR-5681: Make the processing of Collection API calls multi-threaded. (Anshum Gupta) * SOLR-5495: Recovery strategy for leader partitioned from replica case. Hardening diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestFoldingMultitermExtrasQuery.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestFoldingMultitermExtrasQuery.java index 75f498f75be..cc93887ffd2 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestFoldingMultitermExtrasQuery.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestFoldingMultitermExtrasQuery.java @@ -17,6 +17,9 @@ package org.apache.solr.analysis; * limitations under the License. */ +import java.io.File; + +import org.apache.commons.io.FileUtils; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; import org.junit.Test; @@ -29,7 +32,9 @@ public class TestFoldingMultitermExtrasQuery extends SolrTestCaseJ4 { @BeforeClass public static void beforeTests() throws Exception { - initCore("solrconfig-icucollate.xml","schema-folding-extra.xml", "analysis-extras/solr"); + File testHome = createTempDir(); + FileUtils.copyDirectory(getFile("analysis-extras/solr"), testHome); + initCore("solrconfig-icucollate.xml","schema-folding-extra.xml", testHome.getAbsolutePath()); int idx = 1; // ICUFoldingFilterFactory diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java index 95a2993c8f0..3a6671931d6 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java @@ -17,16 +17,21 @@ package org.apache.solr.schema; * limitations under the License. */ +import org.apache.commons.io.FileUtils; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; +import java.io.File; + /** * Tests expert options of {@link ICUCollationField}. */ public class TestICUCollationFieldOptions extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - initCore("solrconfig-icucollate.xml","schema-icucollateoptions.xml", "analysis-extras/solr"); + File testHome = createTempDir(); + FileUtils.copyDirectory(getFile("analysis-extras/solr"), testHome); + initCore("solrconfig-icucollate.xml","schema-icucollateoptions.xml", testHome.getAbsolutePath()); // add some docs assertU(adoc("id", "1", "text", "foo-bar")); assertU(adoc("id", "2", "text", "foo bar")); diff --git a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java index 7abcb10bc56..89390999199 100644 --- a/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java +++ b/solr/contrib/clustering/src/test/org/apache/solr/handler/clustering/AbstractClusteringTestCase.java @@ -16,8 +16,10 @@ package org.apache.solr.handler.clustering; * limitations under the License. */ +import java.io.File; import java.util.Map; +import org.apache.commons.io.FileUtils; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrInputDocument; import org.junit.BeforeClass; @@ -31,7 +33,9 @@ public abstract class AbstractClusteringTestCase extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - initCore("solrconfig.xml", "schema.xml", "clustering/solr"); + File testHome = createTempDir(); + FileUtils.copyDirectory(getFile("clustering/solr"), testHome); + initCore("solrconfig.xml", "schema.xml", testHome.getAbsolutePath()); numberOfDocs = 0; for (String[] doc : DOCUMENTS) { assertNull(h.validateUpdate(adoc("id", Integer.toString(numberOfDocs), "url", doc[0], "title", doc[1], "snippet", doc[2]))); diff --git a/solr/contrib/dataimporthandler/build.xml b/solr/contrib/dataimporthandler/build.xml index a3befd85f82..80bde3693d7 100644 --- a/solr/contrib/dataimporthandler/build.xml +++ b/solr/contrib/dataimporthandler/build.xml @@ -23,9 +23,6 @@ Data Import Handler - - - diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java index 7857ab90cf7..22d4bc8fe59 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java @@ -27,6 +27,7 @@ import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.StandardCharsets; +import java.security.AccessControlException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; @@ -37,6 +38,7 @@ import java.util.Properties; import org.apache.lucene.util.IOUtils; import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrResourceLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** @@ -113,22 +115,34 @@ public class SimplePropertiesWriter extends DIHProperties { configDir = params.get(DIRECTORY); } else { SolrCore core = dataImporter.getCore(); - configDir = (core == null ? "." : core.getResourceLoader().getConfigDir()); + if (core == null) { + configDir = SolrResourceLoader.locateSolrHome(); + } else { + configDir = core.getResourceLoader().getConfigDir(); + } } } private File getPersistFile() { - String filePath = configDir; - if (configDir != null && !configDir.endsWith(File.separator)) filePath += File.separator; - filePath += filename; - return new File(filePath); + final File filePath; + if (new File(filename).isAbsolute() || configDir == null) { + filePath = new File(filename); + } else { + filePath = new File(new File(configDir), filename); + } + return filePath; } + @Override public boolean isWritable() { File persistFile = getPersistFile(); - return persistFile.exists() ? persistFile.canWrite() : persistFile - .getParentFile().canWrite(); - + try { + return persistFile.exists() + ? persistFile.canWrite() + : persistFile.getParentFile().canWrite(); + } catch (AccessControlException e) { + return false; + } } @Override @@ -188,12 +202,7 @@ public class SimplePropertiesWriter extends DIHProperties { Properties newProps = mapToProperties(propObjs); try { existingProps.putAll(newProps); - String filePath = configDir; - if (configDir != null && !configDir.endsWith(File.separator)) { - filePath += File.separator; - } - filePath += filename; - propOutput = new OutputStreamWriter(new FileOutputStream(filePath), StandardCharsets.UTF_8); + propOutput = new OutputStreamWriter(new FileOutputStream(getPersistFile()), StandardCharsets.UTF_8); existingProps.store(propOutput, null); log.info("Wrote last indexed time to " + filename); } catch (Exception e) { diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/PropertyWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/PropertyWriter.java index 811cce044b4..7d3bd1851ac 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/PropertyWriter.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/PropertyWriter.java @@ -1,6 +1,5 @@ package org.apache.solr.handler.dataimport.config; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -23,17 +22,17 @@ import java.util.Map; public class PropertyWriter { private final String type; - private final Map parameters; + private Map parameters; public PropertyWriter(String type, Map parameters) { this.type = type; - this.parameters = Collections.unmodifiableMap(new HashMap<>(parameters)); + this.parameters = new HashMap(parameters); } public Map getParameters() { return parameters; } - + public String getType() { return type; } diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java index 3f3d59670aa..2e3e395226e 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTestCase.java @@ -16,7 +16,17 @@ */ package org.apache.solr.handler.dataimport; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FileUtils; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; @@ -28,18 +38,8 @@ import org.apache.solr.update.MergeIndexesCommand; import org.apache.solr.update.RollbackUpdateCommand; import org.apache.solr.update.processor.UpdateRequestProcessor; import org.apache.solr.update.processor.UpdateRequestProcessorFactory; -import org.apache.solr.common.util.NamedList; -import org.junit.After; import org.junit.Before; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.File; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - /** *

* Abstract base class for DataImportHandler tests @@ -55,27 +55,17 @@ public abstract class AbstractDataImportHandlerTestCase extends // note, a little twisted that we shadow this static method public static void initCore(String config, String schema) throws Exception { - initCore(config, schema, getFile("dih/solr").getAbsolutePath()); + File testHome = createTempDir("core-home"); + FileUtils.copyDirectory(getFile("dih/solr"), testHome); + initCore(config, schema, testHome.getAbsolutePath()); } @Override @Before public void setUp() throws Exception { super.setUp(); - } - - @Override - @After - public void tearDown() throws Exception { - // remove dataimport.properties - File f = new File("solr/collection1/conf/dataimport.properties"); - log.info("Looking for dataimport.properties at: " + f.getAbsolutePath()); - if (f.exists()) { - log.info("Deleting dataimport.properties"); - if (!f.delete()) - log.warn("Could not delete dataimport.properties"); - } - super.tearDown(); + File home = createTempDir("dih-properties"); + System.setProperty("solr.solr.home", home.getAbsolutePath()); } protected String loadDataConfig(String dataConfigFileName) { @@ -103,6 +93,21 @@ public abstract class AbstractDataImportHandlerTestCase extends h.query("/dataimport", request); } + /** + * Redirect {@link SimplePropertiesWriter#filename} to a temporary location + * and return it. + */ + protected File redirectTempProperties(DataImporter di) { + try { + File tempFile = createTempFile(); + di.getConfig().getPropertyWriter().getParameters() + .put(SimplePropertiesWriter.FILENAME, tempFile.getAbsolutePath()); + return tempFile; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + /** * Runs a full-import using the given dataConfig and the provided request parameters. * diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java index d40d41a0f42..e3f18054135 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder.java @@ -73,6 +73,8 @@ public class TestDocBuilder extends AbstractDataImportHandlerTestCase { public void testDeltaImportNoRows_MustNotCommit() { DataImporter di = new DataImporter(); di.loadAndInit(dc_deltaConfig); + redirectTempProperties(di); + DIHConfiguration cfg = di.getConfig(); Entity ent = cfg.getEntities().get(0); MockDataSource.setIterator("select * from x", new ArrayList>().iterator()); diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java index cac2c28d8fd..347a40bf1ab 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFieldReader.java @@ -35,6 +35,8 @@ public class TestFieldReader extends AbstractDataImportHandlerTestCase { public void simple() { DataImporter di = new DataImporter(); di.loadAndInit(config); + redirectTempProperties(di); + TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl(); RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null); List> l = new ArrayList<>(); diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java index c309dcf7fbe..a592c455f4a 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java @@ -72,8 +72,8 @@ public class TestNonWritablePersistFile extends AbstractDataImportHandlerTestCas try { // execute the test only if we are able to set file to read only mode assumeTrue("No dataimport.properties file", f.exists() || f.createNewFile()); - assumeTrue("dataimport.proprties can't be set read only", f.setReadOnly()); - assumeFalse("dataimport.proprties is still writable even though " + + assumeTrue("dataimport.properties can't be set read only", f.setReadOnly()); + assumeFalse("dataimport.properties is still writable even though " + "marked readonly - test running as superuser?", f.canWrite()); ignoreException("Properties is not writable"); diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java index 99608c5d493..5a6ef9dbed7 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestPlainTextEntityProcessor.java @@ -16,11 +16,13 @@ */ package org.apache.solr.handler.dataimport; -import org.junit.Test; - +import java.io.File; +import java.io.IOException; import java.io.StringReader; import java.util.Properties; +import org.junit.Test; + /** * Test for PlainTextEntityProcessor * @@ -30,9 +32,11 @@ import java.util.Properties; */ public class TestPlainTextEntityProcessor extends AbstractDataImportHandlerTestCase { @Test - public void testSimple() { + public void testSimple() throws IOException { DataImporter di = new DataImporter(); di.loadAndInit(DATA_CONFIG); + redirectTempProperties(di); + TestDocBuilder.SolrWriterImpl sw = new TestDocBuilder.SolrWriterImpl(); RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null); di.runCmd(rp, sw); diff --git a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java index a076851353e..cd90d012fde 100644 --- a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java +++ b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java @@ -69,7 +69,7 @@ public class MorphlineBasicMiniMRTest extends SolrTestCaseJ4 { private static final String DOCUMENTS_DIR = RESOURCES_DIR + "/test-documents"; private static final File MINIMR_CONF_DIR = new File(RESOURCES_DIR + "/solr/minimr"); - private static final String SEARCH_ARCHIVES_JAR = JarFinder.getJar(MapReduceIndexerTool.class); + private static String SEARCH_ARCHIVES_JAR; private static MiniDFSCluster dfsCluster = null; private static MiniMRCluster mrCluster = null; @@ -150,6 +150,9 @@ public class MorphlineBasicMiniMRTest extends SolrTestCaseJ4 { System.setProperty("test.build.data", dataDir + File.separator + "hdfs" + File.separator + "build"); System.setProperty("test.cache.data", dataDir + File.separator + "hdfs" + File.separator + "cache"); + // Initialize AFTER test.build.dir is set, JarFinder uses it. + SEARCH_ARCHIVES_JAR = JarFinder.getJar(MapReduceIndexerTool.class); + JobConf conf = new JobConf(); conf.set("dfs.block.access.token.enable", "false"); conf.set("dfs.permissions", "true"); diff --git a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java index 8f47c9413a3..ff3586f1a23 100644 --- a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java +++ b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java @@ -99,7 +99,7 @@ public class MorphlineGoLiveMiniMRTest extends AbstractFullDistribZkTestBase { private static final File MINIMR_INSTANCE_DIR = new File(RESOURCES_DIR + "/solr/minimr"); private static final File MINIMR_CONF_DIR = new File(RESOURCES_DIR + "/solr/minimr"); - private static final String SEARCH_ARCHIVES_JAR = JarFinder.getJar(MapReduceIndexerTool.class); + private static String SEARCH_ARCHIVES_JAR; private static MiniDFSCluster dfsCluster = null; private static MiniMRClientCluster mrCluster = null; @@ -172,6 +172,9 @@ public class MorphlineGoLiveMiniMRTest extends AbstractFullDistribZkTestBase { System.setProperty("test.build.dir", tempDir + File.separator + "hdfs" + File.separator + "test-build-dir"); System.setProperty("test.build.data", tempDir + File.separator + "hdfs" + File.separator + "build"); System.setProperty("test.cache.data", tempDir + File.separator + "hdfs" + File.separator + "cache"); + + // Initialize AFTER test.build.dir is set, JarFinder uses it. + SEARCH_ARCHIVES_JAR = JarFinder.getJar(MapReduceIndexerTool.class); dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null); FileSystem fileSystem = dfsCluster.getFileSystem(); diff --git a/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java b/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java index 3318713045a..90508802d9c 100644 --- a/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java +++ b/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java @@ -133,13 +133,16 @@ public class VelocityResponseWriter implements QueryResponseWriter { // TODO: Externalize Velocity properties String propFile = request.getParams().get("v.properties"); try { - if (propFile == null) - engine.init(); - else { + Properties props = new Properties(); + // Don't create a separate velocity log file by default. + props.put(RuntimeConstants.RUNTIME_LOG, ""); + + if (propFile == null) { + engine.init(props); + } else { InputStream is = null; try { is = resourceLoader.getResourceStream(propFile); - Properties props = new Properties(); props.load(new InputStreamReader(is, StandardCharsets.UTF_8)); engine.init(props); } diff --git a/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java b/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java index e6f389aee67..f06ff1b4b34 100644 --- a/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java +++ b/solr/contrib/velocity/src/test/org/apache/solr/velocity/VelocityResponseWriterTest.java @@ -22,12 +22,10 @@ import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.response.VelocityResponseWriter; import org.apache.solr.request.SolrQueryRequest; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import java.io.StringWriter; -import java.io.IOException; public class VelocityResponseWriterTest extends SolrTestCaseJ4 { @BeforeClass diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java index d2229b5c51c..52cd7b1ebc2 100644 --- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java @@ -245,6 +245,7 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin, public String getDataHome(CoreDescriptor cd) throws IOException { // by default, we go off the instance directory - return normalize(SolrResourceLoader.normalizeDir(cd.getInstanceDir()) + cd.getDataDir()); + String instanceDir = new File(cd.getInstanceDir()).getAbsolutePath(); + return normalize(SolrResourceLoader.normalizeDir(instanceDir) + cd.getDataDir()); } } diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 9d5127809b5..0c98eed2a61 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -161,6 +161,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable { private final SolrResourceLoader resourceLoader; private volatile IndexSchema schema; private final String dataDir; + private final String ulogDir; private final UpdateHandler updateHandler; private final SolrCoreState solrCoreState; @@ -242,6 +243,10 @@ public final class SolrCore implements SolrInfoMBean, Closeable { return dataDir; } + public String getUlogDir() { + return ulogDir; + } + public String getIndexDir() { synchronized (searcherLock) { if (_searcher == null) return getNewIndexDir(); @@ -654,6 +659,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable { this.setName(name); this.schema = null; this.dataDir = null; + this.ulogDir = null; this.solrConfig = null; this.startTime = System.currentTimeMillis(); this.maxWarmingSearchers = 2; // we don't have a config yet, just pick a number. @@ -687,7 +693,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable { if (updateHandler == null) { initDirectoryFactory(); } - + if (dataDir == null) { if (cd.usingDefaultDataDir()) dataDir = config.getDataDir(); if (dataDir == null) { @@ -701,8 +707,18 @@ public final class SolrCore implements SolrInfoMBean, Closeable { } } } - dataDir = SolrResourceLoader.normalizeDir(dataDir); + + String updateLogDir = cd.getUlogDir(); + if (updateLogDir == null) { + updateLogDir = dataDir; + if (new File(updateLogDir).isAbsolute() == false) { + updateLogDir = SolrResourceLoader.normalizeDir(cd.getInstanceDir()) + updateLogDir; + } + } + ulogDir = updateLogDir; + + log.info(logid+"Opening new SolrCore at " + resourceLoader.getInstanceDir() + ", dataDir="+dataDir); if (null != cd && null != cd.getCloudDescriptor()) { diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java b/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java index 6d4ddc7061e..6f3f035a13e 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/SolrSuggester.java @@ -17,6 +17,7 @@ package org.apache.solr.spelling.suggest; * limitations under the License. */ +import java.io.Closeable; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -28,6 +29,7 @@ import org.apache.lucene.search.suggest.Lookup.LookupResult; import org.apache.lucene.search.suggest.Lookup; import org.apache.lucene.util.IOUtils; import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CloseHook; import org.apache.solr.core.SolrCore; import org.apache.solr.search.SolrIndexSearcher; import org.slf4j.Logger; @@ -101,7 +103,22 @@ public class SolrSuggester { // initialize appropriate lookup instance factory = core.getResourceLoader().newInstance(lookupImpl, LookupFactory.class); lookup = factory.create(config, core); - + core.addCloseHook(new CloseHook() { + @Override + public void preClose(SolrCore core) { + if (lookup != null && lookup instanceof Closeable) { + try { + ((Closeable) lookup).close(); + } catch (IOException e) { + LOG.warn("Could not close the suggester lookup.", e); + } + } + } + + @Override + public void postClose(SolrCore core) {} + }); + // if store directory is provided make it or load up the lookup with its content if (store != null) { storeDir = new File(store); diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java index 8c1293a3680..a7c99648a69 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java @@ -17,6 +17,7 @@ package org.apache.solr.spelling.suggest; +import java.io.Closeable; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -33,13 +34,14 @@ import org.apache.lucene.search.spell.Dictionary; import org.apache.lucene.search.spell.HighFrequencyDictionary; import org.apache.lucene.search.spell.SuggestMode; import org.apache.lucene.search.suggest.FileDictionary; -import org.apache.lucene.search.suggest.Lookup.LookupResult; import org.apache.lucene.search.suggest.Lookup; +import org.apache.lucene.search.suggest.Lookup.LookupResult; import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester; import org.apache.lucene.search.suggest.fst.WFSTCompletionLookup; import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.IOUtils; import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CloseHook; import org.apache.solr.core.SolrCore; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.spelling.SolrSpellChecker; @@ -103,6 +105,22 @@ public class Suggester extends SolrSpellChecker { factory = core.getResourceLoader().newInstance(lookupImpl, LookupFactory.class); lookup = factory.create(config, core); + core.addCloseHook(new CloseHook() { + @Override + public void preClose(SolrCore core) { + if (lookup != null && lookup instanceof Closeable) { + try { + ((Closeable) lookup).close(); + } catch (IOException e) { + LOG.warn("Could not close the suggester lookup.", e); + } + } + } + + @Override + public void postClose(SolrCore core) {} + }); + String store = (String)config.get(STORE_DIR); if (store != null) { storeDir = new File(store); @@ -120,6 +138,7 @@ public class Suggester extends SolrSpellChecker { } } } + return name; } diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java index 070fb839cee..1e31716ba28 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java @@ -83,6 +83,9 @@ public class AnalyzingInfixLookupFactory extends LookupFactory { String indexPath = params.get(INDEX_PATH) != null ? params.get(INDEX_PATH).toString() : DEFAULT_INDEX_PATH; + if (new File(indexPath).isAbsolute() == false) { + indexPath = core.getDataDir() + File.separator + indexPath; + } int minPrefixChars = params.get(MIN_PREFIX_CHARS) != null ? Integer.parseInt(params.get(MIN_PREFIX_CHARS).toString()) diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java index b895ff0de72..129a8e27d3e 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java @@ -82,6 +82,9 @@ public class BlendedInfixLookupFactory extends AnalyzingInfixLookupFactory { String indexPath = params.get(INDEX_PATH) != null ? params.get(INDEX_PATH).toString() : DEFAULT_INDEX_PATH; + if (new File(indexPath).isAbsolute() == false) { + indexPath = core.getDataDir() + File.separator + indexPath; + } int minPrefixChars = params.get(MIN_PREFIX_CHARS) != null ? Integer.parseInt(params.get(MIN_PREFIX_CHARS).toString()) diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java index a65012cda26..3f5ed0369cd 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java @@ -222,15 +222,7 @@ public class UpdateLog implements PluginInfoInitialized { * for an existing log whenever the core or update handler changes. */ public void init(UpdateHandler uhandler, SolrCore core) { - // ulogDir from CoreDescriptor overrides - String ulogDir = core.getCoreDescriptor().getUlogDir(); - if (ulogDir != null) { - dataDir = ulogDir; - } - - if (dataDir == null || dataDir.length()==0) { - dataDir = core.getDataDir(); - } + dataDir = core.getUlogDir(); this.uhandler = uhandler; diff --git a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java index f3be1475058..f5009f32b91 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java @@ -79,9 +79,8 @@ public class TestMiniSolrCloudCluster extends LuceneTestCase { @BeforeClass public static void startup() throws Exception { - String testHome = SolrTestCaseJ4.TEST_HOME(); - miniCluster = new MiniSolrCloudCluster(NUM_SERVERS, null, new File(testHome, "solr-no-core.xml"), - null, null); + File solrXml = new File(SolrTestCaseJ4.TEST_HOME(), "solr-no-core.xml"); + miniCluster = new MiniSolrCloudCluster(NUM_SERVERS, null, createTempDir(), solrXml, null, null); } @AfterClass diff --git a/solr/core/src/test/org/apache/solr/core/TestConfigSets.java b/solr/core/src/test/org/apache/solr/core/TestConfigSets.java index ae249fd3ff9..a1ab19d90b2 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfigSets.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfigSets.java @@ -56,17 +56,19 @@ public class TestConfigSets extends SolrTestCaseJ4 { @Test public void testConfigSetServiceFindsConfigSets() { CoreContainer container = null; + SolrCore core1 = null; try { container = setupContainer(getFile("solr/configsets").getAbsolutePath()); - String testDirectory = container.getResourceLoader().getInstanceDir(); + String testDirectory = new File(container.getResourceLoader().getInstanceDir()).getAbsolutePath(); - SolrCore core1 = container.create("core1", testDirectory + "/core1", "configSet", "configset-2"); + core1 = container.create("core1", testDirectory + "core1", "configSet", "configset-2"); assertThat(core1.getCoreDescriptor().getName(), is("core1")); - assertThat(core1.getDataDir(), is(testDirectory + "/core1" + File.separator + "data" + File.separator)); - core1.close(); - + assertThat(core1.getDataDir(), is(testDirectory + "core1" + File.separator + "data" + File.separator)); } finally { + if (core1 != null) { + core1.close(); + } if (container != null) container.shutdown(); } diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java index 23eecc243a6..198dd9ff252 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java @@ -114,12 +114,12 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { // System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory"); // For manual testing only // useFactory(null); // force an FS factory. - master = new SolrInstance("master", null); + master = new SolrInstance(createTempDir("solr-instance"), "master", null); master.setUp(); masterJetty = createJetty(master); masterClient = createNewSolrServer(masterJetty.getLocalPort()); - slave = new SolrInstance("slave", masterJetty.getLocalPort()); + slave = new SolrInstance(createTempDir("solr-instance"), "slave", masterJetty.getLocalPort()); slave.setUp(); slaveJetty = createJetty(slave); slaveClient = createNewSolrServer(slaveJetty.getLocalPort()); @@ -323,7 +323,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { JettySolrRunner repeaterJetty = null; SolrServer repeaterClient = null; try { - repeater = new SolrInstance("repeater", masterJetty.getLocalPort()); + repeater = new SolrInstance(createTempDir("solr-instance"), "repeater", masterJetty.getLocalPort()); repeater.setUp(); repeaterJetty = createJetty(repeater); repeaterClient = createNewSolrServer(repeaterJetty.getLocalPort()); @@ -911,7 +911,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { slaveClient = createNewSolrServer(slaveJetty.getLocalPort()); try { - repeater = new SolrInstance("repeater", null); + repeater = new SolrInstance(createTempDir("solr-instance"), "repeater", null); repeater.setUp(); repeater.copyConfigFile(CONF_DIR + "solrconfig-repeater.xml", "solrconfig.xml"); @@ -1647,13 +1647,15 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { private File dataDir; /** - * @param name used to pick new solr home dir, as well as which + * @param homeDir Base directory to build solr configuration and index in + * @param name used to pick which * "solrconfig-${name}.xml" file gets copied * to solrconfig.xml in new conf dir. * @param testPort if not null, used as a replacement for * TEST_PORT in the cloned config files. */ - public SolrInstance(String name, Integer testPort) { + public SolrInstance(File homeDir, String name, Integer testPort) { + this.homeDir = homeDir; this.name = name; this.testPort = testPort; } @@ -1687,11 +1689,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { System.setProperty("solr.test.sys.prop1", "propone"); System.setProperty("solr.test.sys.prop2", "proptwo"); - File home = new File(dataDir, - getClass().getName() + "-" + - System.currentTimeMillis()); - - homeDir = new File(home, name); dataDir = new File(homeDir + "/collection1", "data"); confDir = new File(homeDir + "/collection1", "conf"); diff --git a/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java b/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java index 373bda78999..d973046eecd 100644 --- a/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java +++ b/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java @@ -16,6 +16,7 @@ package org.apache.solr.rest; * limitations under the License. */ +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -66,12 +67,12 @@ public class TestManagedResourceStorage extends AbstractZkTestCase { * Runs persisted managed resource creation and update tests on JSON storage. */ @Test - public void testFileBasedJsonStorage() throws Exception { - SolrResourceLoader loader = new SolrResourceLoader("./"); - // Solr unit tests can only write to their working directory due to - // a custom Java Security Manager installed in the test environment + public void testFileBasedJsonStorage() throws Exception { + File instanceDir = createTempDir("json-storage"); + SolrResourceLoader loader = new SolrResourceLoader(instanceDir.getAbsolutePath()); NamedList initArgs = new NamedList<>(); - initArgs.add(ManagedResourceStorage.STORAGE_DIR_INIT_ARG, "./managed"); + String managedDir = instanceDir.getAbsolutePath() + File.separator + "managed"; + initArgs.add(ManagedResourceStorage.STORAGE_DIR_INIT_ARG, managedDir); FileStorageIO fileStorageIO = new FileStorageIO(); fileStorageIO.configure(loader, initArgs); doStorageTests(loader, fileStorageIO); diff --git a/solr/core/src/test/org/apache/solr/spelling/suggest/TestAnalyzeInfixSuggestions.java b/solr/core/src/test/org/apache/solr/spelling/suggest/TestAnalyzeInfixSuggestions.java index 87e0dac8130..01f7e033391 100644 --- a/solr/core/src/test/org/apache/solr/spelling/suggest/TestAnalyzeInfixSuggestions.java +++ b/solr/core/src/test/org/apache/solr/spelling/suggest/TestAnalyzeInfixSuggestions.java @@ -1,10 +1,7 @@ package org.apache.solr.spelling.suggest; -import java.io.File; - import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.params.SpellingParams; -import org.junit.AfterClass; import org.junit.BeforeClass; /* @@ -33,16 +30,6 @@ public class TestAnalyzeInfixSuggestions extends SolrTestCaseJ4 { assertQ(req("qt", URI_DEFAULT, "q", "", SpellingParams.SPELLCHECK_BUILD, "true")); } - @AfterClass - public static void afterClass() throws Exception { - File indexPathDir = new File("analyzingInfixSuggesterIndexDir"); - File indexPathDirTmp = new File("analyzingInfixSuggesterIndexDir.tmp"); - if (indexPathDir.exists()) - recurseDelete(indexPathDir); - if (indexPathDirTmp.exists()) - recurseDelete(indexPathDirTmp); - } - public void testSingle() throws Exception { assertQ(req("qt", URI_DEFAULT, "q", "japan", SpellingParams.SPELLCHECK_COUNT, "1"), diff --git a/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java b/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java index 23a690f120e..31620b6931b 100644 --- a/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java +++ b/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java @@ -17,10 +17,7 @@ package org.apache.solr.spelling.suggest; * limitations under the License. */ -import java.io.File; - import org.apache.solr.SolrTestCaseJ4; -import org.junit.AfterClass; import org.junit.BeforeClass; public class TestBlendedInfixSuggestions extends SolrTestCaseJ4 { @@ -31,17 +28,7 @@ public class TestBlendedInfixSuggestions extends SolrTestCaseJ4 { initCore("solrconfig-phrasesuggest.xml","schema-phrasesuggest.xml"); assertQ(req("qt", URI, "q", "", SuggesterParams.SUGGEST_BUILD_ALL, "true")); } - - @AfterClass - public static void afterClass() throws Exception { - File indexPathDir = new File("blendedInfixSuggesterIndexDir"); - File indexPathDirTmp = new File("blendedInfixSuggesterIndexDir.tmp"); - if (indexPathDir.exists()) - recurseDelete(indexPathDir); - if (indexPathDirTmp.exists()) - recurseDelete(indexPathDirTmp); - } - + public void testLinearBlenderType() { assertQ(req("qt", URI, "q", "the", SuggesterParams.SUGGEST_COUNT, "10", SuggesterParams.SUGGEST_DICT, "blended_infix_suggest_linear"), "//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/int[@name='numFound'][.='3']", @@ -55,7 +42,6 @@ public class TestBlendedInfixSuggestions extends SolrTestCaseJ4 { "//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/arr[@name='suggestions']/lst[3]/long[@name='weight'][.='7']", "//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='payload'][.='star']" ); - } public void testReciprocalBlenderType() { @@ -97,5 +83,4 @@ public class TestBlendedInfixSuggestions extends SolrTestCaseJ4 { "//lst[@name='suggest']/lst[@name='blended_infix_suggest_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='payload'][.='star']" ); } - } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java b/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java index c6f4313a518..dc7f82bb3b5 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java @@ -92,7 +92,7 @@ public class TestLBHttpSolrServer extends SolrTestCaseJ4 { httpClient = HttpClientUtil.createClient(null); HttpClientUtil.setConnectionTimeout(httpClient, 1000); for (int i = 0; i < solr.length; i++) { - solr[i] = new SolrInstance("solr/collection1" + i, 0); + solr[i] = new SolrInstance("solr/collection1" + i, createTempDir("instance-" + i), 0); solr[i].setUp(); solr[i].startJetty(); addDocs(solr[i]); @@ -250,9 +250,13 @@ public class TestLBHttpSolrServer extends SolrTestCaseJ4 { int port; JettySolrRunner jetty; - public SolrInstance(String name, int port) { + public SolrInstance(String name, File homeDir, int port) { this.name = name; + this.homeDir = homeDir; this.port = port; + + dataDir = new File(homeDir + "/collection1", "data"); + confDir = new File(homeDir + "/collection1", "conf"); } public String getHomeDir() { @@ -285,12 +289,6 @@ public class TestLBHttpSolrServer extends SolrTestCaseJ4 { public void setUp() throws Exception { - File home = new File(dataDir, - getClass().getName() + "-" + System.currentTimeMillis()); - homeDir = new File(home, name); - dataDir = new File(homeDir + "/collection1", "data"); - confDir = new File(homeDir + "/collection1", "conf"); - homeDir.mkdirs(); dataDir.mkdirs(); confDir.mkdirs(); @@ -301,7 +299,6 @@ public class TestLBHttpSolrServer extends SolrTestCaseJ4 { FileUtils.copyFile(SolrTestCaseJ4.getFile(getSolrConfigFile()), f); f = new File(confDir, "schema.xml"); FileUtils.copyFile(SolrTestCaseJ4.getFile(getSchemaFile()), f); - } public void tearDown() throws Exception { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/SolrPingTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/SolrPingTest.java index 0ed1cf25530..b2f637517b2 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/SolrPingTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/SolrPingTest.java @@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.request; import junit.framework.Assert; +import org.apache.commons.io.FileUtils; import org.apache.solr.SolrJettyTestBase; import org.apache.solr.client.solrj.response.SolrPingResponse; import org.apache.solr.common.SolrException; @@ -27,6 +28,8 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import java.io.File; + /** * Test SolrPing in Solrj */ @@ -34,7 +37,9 @@ public class SolrPingTest extends SolrJettyTestBase { @BeforeClass public static void beforeClass() throws Exception { - initCore("solrconfig.xml", "schema.xml", "solrj/solr", "collection1"); + File testHome = createTempDir(); + FileUtils.copyDirectory(getFile("solrj/solr"), testHome); + initCore("solrconfig.xml", "schema.xml", testHome.getAbsolutePath(), "collection1"); } @Before diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index b98e219b379..7ff5f364fa5 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -48,14 +48,15 @@ public class MiniSolrCloudCluster { * "Mini" SolrCloud cluster to be used for testing * @param numServers number of Solr servers to start * @param hostContext context path of Solr servers used by Jetty + * @param baseDir base directory that the mini cluster should be run from * @param solrXml solr.xml file to be uploaded to ZooKeeper * @param extraServlets Extra servlets to be started by Jetty * @param extraRequestFilters extra filters to be started by Jetty */ - public MiniSolrCloudCluster(int numServers, String hostContext, File solrXml, + public MiniSolrCloudCluster(int numServers, String hostContext, File baseDir, File solrXml, SortedMap extraServlets, SortedMap extraRequestFilters) throws Exception { - testDir = Files.createTempDir(); + testDir = baseDir; String zkDir = testDir.getAbsolutePath() + File.separator + "zookeeper/server1/data";