diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index bd92ea8c3cd..e5002d80a7c 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -188,7 +188,8 @@ Other Changes * LUCENE-6900: Added test for score ordered grouping, and refactored TopGroupsResultTransformer. (David Smiley) - +* SOLR-8336: CoreDescriptor now takes a Path for its instance directory, rather + than a String (Alan Woodward) ================== 5.4.0 ================== diff --git a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java index e34b32ffe8b..775e8202e9d 100644 --- a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java +++ b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java @@ -25,10 +25,10 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; -import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; +import com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FileSystem; @@ -158,13 +158,8 @@ class SolrRecordWriter extends RecordWriter { CoreContainer container = new CoreContainer(loader); container.load(); - - Properties props = new Properties(); - props.setProperty(CoreDescriptor.CORE_DATADIR, dataDirStr); - - CoreDescriptor descr = new CoreDescriptor(container, "core1", solrHomeDir.toString(), props); - - SolrCore core = container.create(descr); + + SolrCore core = container.create("core1", ImmutableMap.of(CoreDescriptor.CORE_DATADIR, dataDirStr)); if (!(core.getDirectoryFactory() instanceof HdfsDirectoryFactory)) { throw new UnsupportedOperationException( diff --git a/solr/core/src/java/org/apache/solr/cloud/CloudConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/CloudConfigSetService.java index da73c8932fa..c371c749796 100644 --- a/solr/core/src/java/org/apache/solr/cloud/CloudConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/cloud/CloudConfigSetService.java @@ -17,8 +17,6 @@ package org.apache.solr.cloud; -import java.nio.file.Paths; - import org.apache.solr.core.ConfigSetService; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.SolrResourceLoader; @@ -37,7 +35,7 @@ public class CloudConfigSetService extends ConfigSetService { // TODO: Shouldn't the collection node be created by the Collections API? zkController.createCollectionZkNode(cd.getCloudDescriptor()); String configName = zkController.getZkStateReader().readConfigName(cd.getCollectionName()); - return new ZkSolrResourceLoader(Paths.get(cd.getInstanceDir()), configName, parentLoader.getClassLoader(), + return new ZkSolrResourceLoader(cd.getInstanceDir(), configName, parentLoader.getClassLoader(), cd.getSubstitutableProperties(), zkController); } diff --git a/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java b/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java index 31e2b8f4e65..dbf33af4aeb 100644 --- a/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java +++ b/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java @@ -17,31 +17,24 @@ package org.apache.solr.cloud; * limitations under the License. */ +import java.util.HashMap; +import java.util.Map; import java.util.Properties; +import com.google.common.base.Strings; import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.util.PropertiesUtil; -import com.google.common.base.Strings; - public class CloudDescriptor { private final CoreDescriptor cd; private String shardId; private String collectionName; - private SolrParams params; private String roles = null; private Integer numShards; private String nodeName = null; - - /* shardRange and shardState are used once-only during sub shard creation for shard splits - * Use the values from {@link Slice} instead */ - volatile String shardRange = null; - volatile Slice.State shardState = Slice.State.ACTIVE; - volatile String shardParent = null; + private Map collectionParams = new HashMap<>(); private volatile boolean isLeader = false; @@ -64,6 +57,12 @@ public class CloudDescriptor { if (Strings.isNullOrEmpty(nodeName)) this.nodeName = null; this.numShards = PropertiesUtil.toInteger(props.getProperty(CloudDescriptor.NUM_SHARDS), null); + + for (String propName : props.stringPropertyNames()) { + if (propName.startsWith(ZkController.COLLECTION_PARAM_PREFIX)) { + collectionParams.put(propName.substring(ZkController.COLLECTION_PARAM_PREFIX.length()), props.getProperty(propName)); + } + } } public Replica.State getLastPublished() { @@ -115,12 +114,8 @@ public class CloudDescriptor { } /** Optional parameters that can change how a core is created. */ - public SolrParams getParams() { - return params; - } - - public void setParams(SolrParams params) { - this.params = params; + public Map getParams() { + return collectionParams; } // setting only matters on core creation diff --git a/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java b/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java index da6850d7c34..1ab51bb6850 100644 --- a/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java +++ b/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java @@ -72,12 +72,11 @@ public class CloudUtil { cc.unload(desc.getName()); } - File instanceDir = new File(desc.getInstanceDir()); try { - FileUtils.deleteDirectory(instanceDir); + FileUtils.deleteDirectory(desc.getInstanceDir().toFile()); } catch (IOException e) { SolrException.log(log, "Failed to delete instance dir for core:" - + desc.getName() + " dir:" + instanceDir.getAbsolutePath()); + + desc.getName() + " dir:" + desc.getInstanceDir()); } log.error("", new SolrException(ErrorCode.SERVER_ERROR, "Will not load SolrCore " + desc.getName() diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 2507d751027..b153d1af5a5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -25,19 +25,7 @@ import java.net.URLEncoder; import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Properties; -import java.util.Set; +import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -52,25 +40,7 @@ import org.apache.solr.cloud.overseer.OverseerAction; import org.apache.solr.cloud.overseer.SliceMutator; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.BeforeReconnect; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.ClusterStateUtil; -import org.apache.solr.common.cloud.DefaultConnectionStrategy; -import org.apache.solr.common.cloud.DefaultZkACLProvider; -import org.apache.solr.common.cloud.DefaultZkCredentialsProvider; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.OnReconnect; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkACLProvider; -import org.apache.solr.common.cloud.ZkCmdExecutor; -import org.apache.solr.common.cloud.ZkConfigManager; -import org.apache.solr.common.cloud.ZkCoreNodeProps; -import org.apache.solr.common.cloud.ZkCredentialsProvider; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.cloud.ZooKeeperException; +import org.apache.solr.common.cloud.*; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.SolrParams; @@ -102,11 +72,11 @@ import org.slf4j.MDC; import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.CORE_NODE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.ELECTION_NODE_PROP; import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP; import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP; import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.CORE_NODE_NAME_PROP; /** * Handle ZooKeeper interactions. @@ -1337,7 +1307,6 @@ public final class ZkController { try { if (!zkClient.exists(collectionPath, true)) { log.info("Creating collection in ZooKeeper:" + collection); - SolrParams params = cd.getParams(); try { Map collectionProps = new HashMap<>(); @@ -1346,15 +1315,8 @@ public final class ZkController { String defaultConfigName = System.getProperty(COLLECTION_PARAM_PREFIX + CONFIGNAME_PROP, collection); // params passed in - currently only done via core admin (create core commmand). - if (params != null) { - Iterator iter = params.getParameterNamesIterator(); - while (iter.hasNext()) { - String paramName = iter.next(); - if (paramName.startsWith(COLLECTION_PARAM_PREFIX)) { - collectionProps.put(paramName.substring(COLLECTION_PARAM_PREFIX.length()), params.get(paramName)); - } - } - + if (cd.getParams().size() > 0) { + collectionProps.putAll(cd.getParams()); // if the config name wasn't passed in, use the default if (!collectionProps.containsKey(CONFIGNAME_PROP)) { // TODO: getting the configName from the collectionPath should fail since we already know it doesn't exist? @@ -1795,8 +1757,7 @@ public final class ZkController { String confName = cd.getCollectionName(); if (StringUtils.isEmpty(confName)) confName = coreName; - String instanceDir = cd.getInstanceDir(); - Path udir = Paths.get(instanceDir).resolve("conf"); + Path udir = cd.getInstanceDir().resolve("conf"); log.info("Uploading directory " + udir + " with name " + confName + " for SolrCore " + coreName); configManager.uploadConfigDir(udir, confName); } @@ -2524,4 +2485,22 @@ public final class ZkController { super(code, msg); } } + + public boolean checkIfCoreNodeNameAlreadyExists(CoreDescriptor dcore) { + DocCollection collection = zkStateReader.getClusterState().getCollectionOrNull(dcore.getCollectionName()); + if (collection != null) { + Collection slices = collection.getSlices(); + + for (Slice slice : slices) { + Collection replicas = slice.getReplicas(); + for (Replica replica : replicas) { + if (replica.getName().equals( + dcore.getCloudDescriptor().getCoreNodeName())) { + return true; + } + } + } + } + return false; + } } diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java index e63e7b5a803..8ac315cc87e 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java @@ -167,7 +167,7 @@ public abstract class ConfigSetService { protected Path locateInstanceDir(CoreDescriptor cd) { String configSet = cd.getConfigSet(); if (configSet == null) - return Paths.get(cd.getInstanceDir()); + return cd.getInstanceDir(); Path configSetDirectory = configSetBase.resolve(configSet); if (!Files.isDirectory(configSetDirectory)) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index b042d0c7132..56b69cfee42 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -36,6 +36,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import org.apache.solr.client.solrj.impl.HttpClientConfigurer; import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; @@ -518,7 +519,7 @@ public class CoreContainer { } private static void checkForDuplicateCoreNames(List cds) { - Map addedCores = Maps.newHashMap(); + Map addedCores = Maps.newHashMap(); for (CoreDescriptor cd : cds) { final String name = cd.getName(); if (addedCores.containsKey(name)) @@ -707,12 +708,82 @@ public class CoreContainer { } /** - * Creates a new core based on a CoreDescriptor, publishing the core state to the cluster - * @param cd the CoreDescriptor + * Creates a new core, publishing the core state to the cluster + * @param coreName the core name + * @param parameters the core parameters * @return the newly created core */ - public SolrCore create(CoreDescriptor cd) { - return create(cd, true); + public SolrCore create(String coreName, Map parameters) { + return create(coreName, cfg.getCoreRootDirectory().resolve(coreName), parameters); + } + + /** + * Creates a new core in a specified instance directory, publishing the core state to the cluster + * @param coreName the core name + * @param instancePath the instance directory + * @param parameters the core parameters + * @return the newly created core + */ + public SolrCore create(String coreName, Path instancePath, Map parameters) { + + CoreDescriptor cd = new CoreDescriptor(this, coreName, instancePath, parameters); + + // TODO: There's a race here, isn't there? + if (getAllCoreNames().contains(coreName)) { + log.warn("Creating a core with existing name is not allowed"); + // TODO: Shouldn't this be a BAD_REQUEST? + throw new SolrException(ErrorCode.SERVER_ERROR, "Core with name '" + coreName + "' already exists."); + } + + boolean preExisitingZkEntry = false; + try { + if (getZkController() != null) { + if (!Overseer.isLegacy(getZkController().getZkStateReader().getClusterProps())) { + if (cd.getCloudDescriptor().getCoreNodeName() == null) { + throw new SolrException(ErrorCode.SERVER_ERROR, "non legacy mode coreNodeName missing " + parameters.toString()); + + } + } + preExisitingZkEntry = getZkController().checkIfCoreNodeNameAlreadyExists(cd); + } + + SolrCore core = create(cd, true); + + // only write out the descriptor if the core is successfully created + coresLocator.create(this, cd); + + return core; + } + catch (Exception ex) { + if (isZooKeeperAware() && !preExisitingZkEntry) { + try { + getZkController().unregister(coreName, cd); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + SolrException.log(log, null, e); + } catch (KeeperException e) { + SolrException.log(log, null, e); + } + } + + Throwable tc = ex; + Throwable c = null; + do { + tc = tc.getCause(); + if (tc != null) { + c = tc; + } + } while (tc != null); + + String rootMsg = ""; + if (c != null) { + rootMsg = " Caused by: " + c.getMessage(); + } + + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "Error CREATEing SolrCore '" + coreName + "': " + ex.getMessage() + rootMsg, ex); + } + } /** @@ -723,7 +794,7 @@ public class CoreContainer { * * @return the newly created core */ - public SolrCore create(CoreDescriptor dcore, boolean publishState) { + private SolrCore create(CoreDescriptor dcore, boolean publishState) { if (isShutDown) { throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Solr has been shutdown."); @@ -960,8 +1031,8 @@ public class CoreContainer { return null; } - public String getCoreRootDirectory() { - return cfg.getCoreRootDirectory().toString(); + public Path getCoreRootDirectory() { + return cfg.getCoreRootDirectory(); } /** @@ -1007,7 +1078,7 @@ public class CoreContainer { if (zkSys.getZkController() != null) { zkSys.getZkController().throwErrorIfReplicaReplaced(desc); } - core = create(desc); // This should throw an error if it fails. + core = create(desc, true); // This should throw an error if it fails. } core.open(); } diff --git a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java index c2e70a13f14..22e6497061a 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java +++ b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java @@ -17,26 +17,26 @@ package org.apache.solr.core; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.commons.lang.StringUtils; import org.apache.solr.cloud.CloudDescriptor; import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.IOUtils; import org.apache.solr.util.PropertiesUtil; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; -import java.util.Locale; -import java.util.Properties; - -import static com.google.common.base.Preconditions.checkNotNull; - /** * A Solr core descriptor * @@ -47,8 +47,6 @@ public class CoreDescriptor { // Properties file name constants public static final String CORE_NAME = "name"; public static final String CORE_CONFIG = "config"; - public static final String CORE_INSTDIR = "instanceDir"; - public static final String CORE_ABS_INSTDIR = "absoluteInstDir"; public static final String CORE_DATADIR = "dataDir"; public static final String CORE_ULOGDIR = "ulogDir"; public static final String CORE_SCHEMA = "schema"; @@ -91,13 +89,12 @@ public class CoreDescriptor { .build(); private static ImmutableList requiredProperties = ImmutableList.of( - CORE_NAME, CORE_INSTDIR, CORE_ABS_INSTDIR + CORE_NAME ); public static ImmutableList standardPropNames = ImmutableList.of( CORE_NAME, CORE_CONFIG, - CORE_INSTDIR, CORE_DATADIR, CORE_ULOGDIR, CORE_SCHEMA, @@ -118,6 +115,8 @@ public class CoreDescriptor { private final CloudDescriptor cloudDesc; + private final Path instanceDir; + /** The original standard core properties, before substitution */ protected final Properties originalCoreProperties = new Properties(); @@ -130,61 +129,72 @@ public class CoreDescriptor { /** The properties for this core, substitutable by resource loaders */ protected final Properties substitutableProperties = new Properties(); - /** - * Create a new CoreDescriptor. - * @param container the CoreDescriptor's container - * @param name the CoreDescriptor's name - * @param instanceDir a String containing the instanceDir - * @param coreProps a Properties object of the properties for this core - */ - public CoreDescriptor(CoreContainer container, String name, String instanceDir, - Properties coreProps) { - this(container, name, instanceDir, coreProps, null); + public CoreDescriptor(CoreContainer container, String name, Path instanceDir, String... properties) { + this(container, name, instanceDir, toMap(properties)); } - public CoreDescriptor(CoreContainer container, String name, String instanceDir, String... properties) { - this(container, name, instanceDir, toProperties(properties)); - } - - private static Properties toProperties(String... properties) { - Properties props = new Properties(); + private static Map toMap(String... properties) { + Map props = new HashMap<>(); assert properties.length % 2 == 0; for (int i = 0; i < properties.length; i += 2) { - props.setProperty(properties[i], properties[i+1]); + props.put(properties[i], properties[i+1]); } return props; } - + + /** + * Create a new CoreDescriptor with a given name and instancedir + * @param container the CoreDescriptor's container + * @param name the CoreDescriptor's name + * @param instanceDir the CoreDescriptor's instancedir + */ + public CoreDescriptor(CoreContainer container, String name, Path instanceDir) { + this(container, name, instanceDir, Collections.emptyMap()); + } + + /** + * Create a new CoreDescriptor using the properties of an existing one + * @param coreName the new CoreDescriptor's name + * @param other the CoreDescriptor to copy + */ + public CoreDescriptor(String coreName, CoreDescriptor other) { + this.coreContainer = other.coreContainer; + this.cloudDesc = other.cloudDesc; + this.instanceDir = other.instanceDir; + this.originalExtraProperties.putAll(other.originalExtraProperties); + this.originalCoreProperties.putAll(other.originalCoreProperties); + this.coreProperties.putAll(other.coreProperties); + this.substitutableProperties.putAll(other.substitutableProperties); + this.coreProperties.setProperty(CORE_NAME, coreName); + this.originalCoreProperties.setProperty(CORE_NAME, coreName); + this.substitutableProperties.setProperty(SOLR_CORE_PROP_PREFIX + CORE_NAME, coreName); + } + /** * Create a new CoreDescriptor. * @param container the CoreDescriptor's container * @param name the CoreDescriptor's name - * @param instanceDir a String containing the instanceDir - * @param coreProps a Properties object of the properties for this core - * @param params additional params + * @param instanceDir a Path resolving to the instanceDir + * @param coreProps a Map of the properties for this core */ - public CoreDescriptor(CoreContainer container, String name, String instanceDir, - Properties coreProps, SolrParams params) { + public CoreDescriptor(CoreContainer container, String name, Path instanceDir, + Map coreProps) { this.coreContainer = container; + this.instanceDir = instanceDir; originalCoreProperties.setProperty(CORE_NAME, name); - originalCoreProperties.setProperty(CORE_INSTDIR, instanceDir); Properties containerProperties = container.getContainerProperties(); name = PropertiesUtil.substituteProperty(checkPropertyIsNotEmpty(name, CORE_NAME), containerProperties); - instanceDir = PropertiesUtil.substituteProperty(checkPropertyIsNotEmpty(instanceDir, CORE_INSTDIR), - containerProperties); coreProperties.putAll(defaultProperties); coreProperties.put(CORE_NAME, name); - coreProperties.put(CORE_INSTDIR, instanceDir); - coreProperties.put(CORE_ABS_INSTDIR, convertToAbsolute(instanceDir, container.getCoreRootDirectory())); - for (String propname : coreProps.stringPropertyNames()) { + for (String propname : coreProps.keySet()) { - String propvalue = coreProps.getProperty(propname); + String propvalue = coreProps.get(propname); if (isUserDefinedProperty(propname)) originalExtraProperties.put(propname, propvalue); @@ -202,9 +212,6 @@ public class CoreDescriptor { // TODO maybe make this a CloudCoreDescriptor subclass? if (container.isZooKeeperAware()) { cloudDesc = new CloudDescriptor(name, coreProperties, this); - if (params != null) { - cloudDesc.setParams(params); - } } else { cloudDesc = null; @@ -226,20 +233,16 @@ public class CoreDescriptor { */ protected void loadExtraProperties() { String filename = coreProperties.getProperty(CORE_PROPERTIES, DEFAULT_EXTERNAL_PROPERTIES_FILE); - File propertiesFile = resolvePaths(filename); - if (propertiesFile.exists()) { - FileInputStream in = null; - try { - in = new FileInputStream(propertiesFile); + Path propertiesFile = instanceDir.resolve(filename); + if (Files.exists(propertiesFile)) { + try (InputStream is = Files.newInputStream(propertiesFile)) { Properties externalProps = new Properties(); - externalProps.load(new InputStreamReader(in, StandardCharsets.UTF_8)); + externalProps.load(new InputStreamReader(is, StandardCharsets.UTF_8)); coreProperties.putAll(externalProps); } catch (IOException e) { String message = String.format(Locale.ROOT, "Could not load properties from %s: %s:", - propertiesFile.getAbsoluteFile(), e.toString()); + propertiesFile.toString(), e.toString()); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, message); - } finally { - IOUtils.closeQuietly(in); } } } @@ -258,13 +261,6 @@ public class CoreDescriptor { } } - protected File resolvePaths(String filepath) { - File file = new File(filepath); - if (file.isAbsolute()) - return file; - return new File(getInstanceDir(), filepath); - } - /** * Is this property a Solr-standard property, or is it an extra property * defined per-core by the user? @@ -283,33 +279,6 @@ public class CoreDescriptor { return value; } - /** - * Create a new CoreDescriptor with a given name and instancedir - * @param container the CoreDescriptor's container - * @param name the CoreDescriptor's name - * @param instanceDir the CoreDescriptor's instancedir - */ - public CoreDescriptor(CoreContainer container, String name, String instanceDir) { - this(container, name, instanceDir, new Properties()); - } - - /** - * Create a new CoreDescriptor using the properties of an existing one - * @param coreName the new CoreDescriptor's name - * @param other the CoreDescriptor to copy - */ - public CoreDescriptor(String coreName, CoreDescriptor other) { - this.coreContainer = other.coreContainer; - this.cloudDesc = other.cloudDesc; - this.originalExtraProperties.putAll(other.originalExtraProperties); - this.originalCoreProperties.putAll(other.originalCoreProperties); - this.coreProperties.putAll(other.coreProperties); - this.substitutableProperties.putAll(other.substitutableProperties); - this.coreProperties.setProperty(CORE_NAME, coreName); - this.originalCoreProperties.setProperty(CORE_NAME, coreName); - this.substitutableProperties.setProperty(SOLR_CORE_PROP_PREFIX + CORE_NAME, coreName); - } - public String getPropertiesName() { return coreProperties.getProperty(CORE_PROPERTIES); } @@ -322,22 +291,11 @@ public class CoreDescriptor { return defaultProperties.get(CORE_DATADIR).equals(coreProperties.getProperty(CORE_DATADIR)); } - /**@return the core instance directory. */ - public String getRawInstanceDir() { - return coreProperties.getProperty(CORE_INSTDIR); - } - - private static String convertToAbsolute(String instDir, String solrHome) { - checkNotNull(instDir); - return SolrResourceLoader.normalizeDir(Paths.get(solrHome).resolve(instDir).toString()); - } - /** - * - * @return the core instance directory, prepended with solr_home if not an absolute path. + * @return the core instance directory */ - public String getInstanceDir() { - return coreProperties.getProperty(CORE_ABS_INSTDIR); + public Path getInstanceDir() { + return instanceDir; } /**@return the core configuration resource name. */ @@ -401,12 +359,7 @@ public class CoreDescriptor { @Override public String toString() { - return new StringBuilder("CoreDescriptor[name=") - .append(this.getName()) - .append(";instanceDir=") - .append(this.getInstanceDir()) - .append("]") - .toString(); + return "CoreDescriptor[name=" + this.getName() + ";instanceDir=" + this.getInstanceDir() + "]"; } public String getConfigSet() { diff --git a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java index d48b3c50701..b29d7816b01 100644 --- a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java +++ b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java @@ -28,7 +28,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Properties; import com.google.common.collect.Lists; @@ -158,7 +160,11 @@ public class CorePropertiesLocator implements CoresLocator { try (InputStream fis = Files.newInputStream(propertiesFile)) { coreProperties.load(new InputStreamReader(fis, StandardCharsets.UTF_8)); String name = createName(coreProperties, instanceDir); - return new CoreDescriptor(cc, name, instanceDir.toString(), coreProperties); + Map propMap = new HashMap<>(); + for (String key : coreProperties.stringPropertyNames()) { + propMap.put(key, coreProperties.getProperty(key)); + } + return new CoreDescriptor(cc, name, instanceDir, propMap); } catch (IOException e) { logger.error("Couldn't load core descriptor from {}:{}", propertiesFile, e.toString()); @@ -175,9 +181,6 @@ public class CorePropertiesLocator implements CoresLocator { Properties p = new Properties(); p.putAll(cd.getPersistableStandardProperties()); p.putAll(cd.getPersistableUserProperties()); - // We don't persist the instance directory, as that's defined by the location - // of the properties file. - p.remove(CoreDescriptor.CORE_INSTDIR); return p; } diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java index 8413e12d509..902c860fef7 100644 --- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java @@ -265,8 +265,7 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin, public String getDataHome(CoreDescriptor cd) throws IOException { // by default, we go off the instance directory - String instanceDir = new File(cd.getInstanceDir()).getAbsolutePath(); - return normalize(SolrResourceLoader.normalizeDir(instanceDir) + cd.getDataDir()); + return cd.getInstanceDir().resolve(cd.getDataDir()).toAbsolutePath().toString(); } /** diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 7a15562d3f1..84679156815 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -1006,10 +1006,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable { private String initUpdateLogDir(CoreDescriptor coreDescriptor) { String updateLogDir = coreDescriptor.getUlogDir(); if (updateLogDir == null) { - updateLogDir = dataDir; - if (new File(updateLogDir).isAbsolute() == false) { - updateLogDir = SolrResourceLoader.normalizeDir(coreDescriptor.getInstanceDir()) + updateLogDir; - } + updateLogDir = coreDescriptor.getInstanceDir().resolve(dataDir).normalize().toAbsolutePath().toString(); } return updateLogDir; } @@ -2492,12 +2489,11 @@ public final class SolrCore implements SolrInfoMBean, Closeable { public void postClose(SolrCore core) { CoreDescriptor cd = core.getCoreDescriptor(); if (cd != null) { - File instanceDir = new File(cd.getInstanceDir()); try { - FileUtils.deleteDirectory(instanceDir); + FileUtils.deleteDirectory(cd.getInstanceDir().toFile()); } catch (IOException e) { SolrException.log(log, "Failed to delete instance dir for core:" - + core.getName() + " dir:" + instanceDir.getAbsolutePath()); + + core.getName() + " dir:" + cd.getInstanceDir()); } } } @@ -2516,12 +2512,11 @@ public final class SolrCore implements SolrInfoMBean, Closeable { } } if (deleteInstanceDir) { - File instanceDir = new File(cd.getInstanceDir()); try { - FileUtils.deleteDirectory(instanceDir); + FileUtils.deleteDirectory(cd.getInstanceDir().toFile()); } catch (IOException e) { SolrException.log(log, "Failed to delete instance dir for unloaded core:" + cd.getName() - + " dir:" + instanceDir.getAbsolutePath()); + + " dir:" + cd.getInstanceDir()); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java index 55962a9593c..fa11ad64f93 100644 --- a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java +++ b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java @@ -18,7 +18,6 @@ package org.apache.solr.handler; import java.io.File; import java.io.IOException; -import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; @@ -26,8 +25,6 @@ import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Locale; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.store.Directory; @@ -63,8 +60,7 @@ public class SnapShooter { snapDir = core.getDataDir(); } else { - File base = new File(core.getCoreDescriptor().getInstanceDir()); - snapDir = org.apache.solr.util.FileUtils.resolvePath(base, location).getAbsolutePath(); + snapDir = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString(); } this.snapshotName = snapshotName; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java index 3f64ed75d68..19f78b99600 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java @@ -19,7 +19,16 @@ package org.apache.solr.handler.admin; import java.io.File; import java.io.IOException; -import java.util.*; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -31,7 +40,6 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; import org.apache.solr.cloud.CloudDescriptor; -import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.SyncStrategy; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; @@ -71,6 +79,7 @@ import org.apache.solr.update.processor.UpdateRequestProcessor; import org.apache.solr.update.processor.UpdateRequestProcessorChain; import org.apache.solr.util.DefaultSolrThreadFactory; import org.apache.solr.util.NumberUtils; +import org.apache.solr.util.PropertiesUtil; import org.apache.solr.util.RefCounted; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -547,37 +556,33 @@ public class CoreAdminHandler extends RequestHandlerBase { public static ImmutableMap cloudParamToProp; - protected static CoreDescriptor buildCoreDescriptor(SolrParams params, CoreContainer container) { + protected static Map buildCoreParams(SolrParams params) { - String name = checkNotEmpty(params.get(CoreAdminParams.NAME), - "Missing parameter [" + CoreAdminParams.NAME + "]"); + Map coreParams = new HashMap<>(); - Properties coreProps = new Properties(); + // standard core create parameters for (String param : paramToProp.keySet()) { String value = params.get(param, null); if (StringUtils.isNotEmpty(value)) { - coreProps.setProperty(paramToProp.get(param), value); + coreParams.put(paramToProp.get(param), value); } } + + // extra properties Iterator paramsIt = params.getParameterNamesIterator(); while (paramsIt.hasNext()) { String param = paramsIt.next(); - if (!param.startsWith(CoreAdminParams.PROPERTY_PREFIX)) - continue; - String propName = param.substring(CoreAdminParams.PROPERTY_PREFIX.length()); - String propValue = params.get(param); - coreProps.setProperty(propName, propValue); + if (param.startsWith(CoreAdminParams.PROPERTY_PREFIX)) { + String propName = param.substring(CoreAdminParams.PROPERTY_PREFIX.length()); + String propValue = params.get(param); + coreParams.put(propName, propValue); + } + if (param.startsWith(ZkController.COLLECTION_PARAM_PREFIX)) { + coreParams.put(param, params.get(param)); + } } - String instancedir = params.get(CoreAdminParams.INSTANCE_DIR); - if (StringUtils.isEmpty(instancedir) && coreProps.getProperty(CoreAdminParams.INSTANCE_DIR) != null) { - instancedir = coreProps.getProperty(CoreAdminParams.INSTANCE_DIR); - } else if (StringUtils.isEmpty(instancedir)){ - instancedir = name; // will be resolved later against solr.home - //instancedir = container.getSolrHome() + "/" + name; - } - - return new CoreDescriptor(container, name, instancedir, coreProps, params); + return coreParams; } private static String checkNotEmpty(String value, String message) { @@ -595,88 +600,23 @@ public class CoreAdminHandler extends RequestHandlerBase { SolrParams params = req.getParams(); log.info("core create command {}", params); - CoreDescriptor dcore = buildCoreDescriptor(params, coreContainer); + String coreName = params.required().get(CoreAdminParams.NAME); + Map coreParams = buildCoreParams(params); - if (coreContainer.getAllCoreNames().contains(dcore.getName())) { - log.warn("Creating a core with existing name is not allowed"); - throw new SolrException(ErrorCode.SERVER_ERROR, - "Core with name '" + dcore.getName() + "' already exists."); + Path instancePath = coreContainer.getCoreRootDirectory().resolve(coreName); + + // TODO: Should we nuke setting odd instance paths? They break core discovery, generally + String instanceDir = req.getParams().get(CoreAdminParams.INSTANCE_DIR); + if (instanceDir == null) + instanceDir = req.getParams().get("property.instanceDir"); + if (instanceDir != null) { + instanceDir = PropertiesUtil.substituteProperty(instanceDir, coreContainer.getContainerProperties()); + instancePath = coreContainer.getCoreRootDirectory().resolve(instanceDir).normalize(); } - // TODO this should be moved into CoreContainer, really... - boolean preExisitingZkEntry = false; - try { - if (coreContainer.getZkController() != null) { - if (!Overseer.isLegacy(coreContainer.getZkController().getZkStateReader().getClusterProps())) { - if (dcore.getCloudDescriptor().getCoreNodeName() == null) { - throw new SolrException(ErrorCode.SERVER_ERROR, - "non legacy mode coreNodeName missing " + params); - - } - } - - preExisitingZkEntry = checkIfCoreNodeNameAlreadyExists(dcore); + coreContainer.create(coreName, instancePath, coreParams); - } - - SolrCore core = coreContainer.create(dcore); - - // only write out the descriptor if the core is successfully created - coreContainer.getCoresLocator().create(coreContainer, dcore); - - rsp.add("core", core.getName()); - } - catch (Exception ex) { - if (coreContainer.isZooKeeperAware() && dcore != null && !preExisitingZkEntry) { - try { - coreContainer.getZkController().unregister(dcore.getName(), dcore); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - SolrException.log(log, null, e); - } catch (KeeperException e) { - SolrException.log(log, null, e); - } - } - - Throwable tc = ex; - Throwable c = null; - do { - tc = tc.getCause(); - if (tc != null) { - c = tc; - } - } while (tc != null); - - String rootMsg = ""; - if (c != null) { - rootMsg = " Caused by: " + c.getMessage(); - } - - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Error CREATEing SolrCore '" + dcore.getName() + "': " + - ex.getMessage() + rootMsg, ex); - } - } - - - private boolean checkIfCoreNodeNameAlreadyExists(CoreDescriptor dcore) { - ZkStateReader zkStateReader = coreContainer.getZkController() - .getZkStateReader(); - DocCollection collection = zkStateReader.getClusterState().getCollectionOrNull(dcore.getCollectionName()); - if (collection != null) { - Collection slices = collection.getSlices(); - - for (Slice slice : slices) { - Collection replicas = slice.getReplicas(); - for (Replica replica : replicas) { - if (replica.getName().equals( - dcore.getCloudDescriptor().getCoreNodeName())) { - return true; - } - } - } - } - return false; + rsp.add("core", coreName); } /** diff --git a/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java b/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java index 6ef50aed153..2d9460f5f73 100644 --- a/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java +++ b/solr/core/src/test/org/apache/solr/TestCrossCoreJoin.java @@ -20,9 +20,9 @@ package org.apache.solr; import java.io.StringWriter; import java.util.Collections; +import com.google.common.collect.ImmutableMap; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.SolrCore; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; @@ -49,15 +49,8 @@ public class TestCrossCoreJoin extends SolrTestCaseJ4 { // FileUtils.copyDirectory(getFile("solrj/solr"), testHome); initCore("solrconfig.xml", "schema12.xml", TEST_HOME(), "collection1"); final CoreContainer coreContainer = h.getCoreContainer(); - final CoreDescriptor toCoreDescriptor = coreContainer.getCoreDescriptor("collection1"); - final CoreDescriptor fromCoreDescriptor = new CoreDescriptor("fromCore", toCoreDescriptor) { - @Override - public String getSchemaName() { - return "schema.xml"; - } - }; - fromCore = coreContainer.create(fromCoreDescriptor); + fromCore = coreContainer.create("fromCore", ImmutableMap.of("configSet", "minimal")); assertU(add(doc("id", "1", "name", "john", "title", "Director", "dept_s", "Engineering"))); assertU(add(doc("id", "2", "name", "mark", "title", "VP", "dept_s", "Marketing"))); @@ -108,15 +101,17 @@ public class TestCrossCoreJoin extends SolrTestCaseJ4 { @Test public void testCoresAreDifferent() throws Exception { assertQEx("schema12.xml" + " has no \"cat\" field", req("cat:*"), ErrorCode.BAD_REQUEST); - final LocalSolrQueryRequest req = new LocalSolrQueryRequest(fromCore, "cat:*", "lucene", 0, 100, Collections.emptyMap()); + final LocalSolrQueryRequest req = new LocalSolrQueryRequest(fromCore, "cat:*", "/select", 0, 100, Collections.emptyMap()); final String resp = query(fromCore, req); assertTrue(resp, resp.contains("numFound=\"1\"")); - assertTrue(resp, resp.contains("10")); + assertTrue(resp, resp.contains("10")); } public String query(SolrCore core, SolrQueryRequest req) throws Exception { String handler = "standard"; + if (req.getParams().get("qt") != null) + handler = req.getParams().get("qt"); SolrQueryResponse rsp = new SolrQueryResponse(); SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); core.execute(core.getRequestHandler(handler), req, rsp); diff --git a/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java b/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java index 997fe8e3a68..360d7f8642c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java @@ -17,6 +17,13 @@ package org.apache.solr.cloud; * limitations under the License. */ +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import com.google.common.collect.ImmutableMap; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.ClusterState; @@ -27,8 +34,6 @@ import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.CoreDescriptor; -import org.apache.solr.core.SolrCore; import org.apache.zookeeper.CreateMode; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -36,12 +41,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - @Slow public class ClusterStateUpdateTest extends SolrTestCaseJ4 { protected static Logger log = LoggerFactory @@ -157,10 +156,7 @@ public class ClusterStateUpdateTest extends SolrTestCaseJ4 { CreateMode.PERSISTENT, true); zkClient.close(); - CoreDescriptor dcore = buildCoreDescriptor(container1, "testcore", "testcore") - .withDataDir(dataDir4.getAbsolutePath()).build(); - - SolrCore core = container1.create(dcore); + container1.create("testcore", ImmutableMap.of("dataDir", dataDir4.getAbsolutePath())); ZkController zkController2 = container2.getZkController(); diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTests.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTests.java index 4780167b008..dda78c2ef32 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTests.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTests.java @@ -17,6 +17,17 @@ package org.apache.solr.cloud; * limitations under the License. */ +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + import org.apache.commons.codec.binary.StringUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -37,17 +48,6 @@ import org.apache.solr.util.TimeOut; import org.apache.zookeeper.KeeperException; import org.junit.Test; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - import static org.apache.solr.cloud.ReplicaPropertiesBase.verifyUniqueAcrossCollection; @LuceneTestCase.Slow @@ -232,12 +232,10 @@ public class CollectionsAPISolrJTests extends AbstractFullDistribZkTestBase { String collectionName = "solrj_test_core_props"; File tmpDir = createTempDir("testPropertyParamsForCreate").toFile(); - File instanceDir = new File(tmpDir, "instanceDir-" + TestUtil.randomSimpleString(random(), 1, 5)); File dataDir = new File(tmpDir, "dataDir-" + TestUtil.randomSimpleString(random(), 1, 5)); File ulogDir = new File(tmpDir, "ulogDir-" + TestUtil.randomSimpleString(random(), 1, 5)); Properties properties = new Properties(); - properties.put(CoreAdminParams.INSTANCE_DIR, instanceDir.getAbsolutePath()); properties.put(CoreAdminParams.DATA_DIR, dataDir.getAbsolutePath()); properties.put(CoreAdminParams.ULOG_DIR, ulogDir.getAbsolutePath()); @@ -262,9 +260,6 @@ public class CollectionsAPISolrJTests extends AbstractFullDistribZkTestBase { CoreAdminResponse status = CoreAdminRequest.getStatus(replica1.getStr("core"), client); NamedList coreStatus = status.getCoreStatus(replica1.getStr("core")); String dataDirStr = (String) coreStatus.get("dataDir"); - String instanceDirStr = (String) coreStatus.get("instanceDir"); - assertEquals("Instance dir does not match param passed in property.instanceDir syntax", - new File(instanceDirStr).getAbsolutePath(), instanceDir.getAbsolutePath()); assertEquals("Data dir does not match param given in property.dataDir syntax", new File(dataDirStr).getAbsolutePath(), dataDir.getAbsolutePath()); } diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java index bed3d7280fc..1f944e1386a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java @@ -17,23 +17,10 @@ package org.apache.solr.cloud; * limitations under the License. */ -import org.apache.lucene.util.LuceneTestCase.Slow; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.CoreDescriptor; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.SAXException; - import javax.xml.parsers.ParserConfigurationException; -import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -42,6 +29,21 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.google.common.collect.ImmutableMap; +import org.apache.lucene.util.LuceneTestCase.Slow; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrResourceLoader; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.SAXException; + @Slow public class LeaderElectionIntegrationTest extends SolrTestCaseJ4 { protected static Logger log = LoggerFactory @@ -133,11 +135,11 @@ public class LeaderElectionIntegrationTest extends SolrTestCaseJ4 { private void setupContainer(int port, String shard) throws IOException, ParserConfigurationException, SAXException { - File data = createTempDir().toFile(); + Path data = createTempDir(); System.setProperty("hostPort", Integer.toString(port)); System.setProperty("shard", shard); - System.setProperty("solr.data.dir", data.getAbsolutePath()); + System.setProperty("solr.data.dir", data.toString()); System.setProperty("solr.solr.home", TEST_HOME()); Set ports = shardPorts.get(shard); if (ports == null) { @@ -145,9 +147,12 @@ public class LeaderElectionIntegrationTest extends SolrTestCaseJ4 { shardPorts.put(shard, ports); } ports.add(port); - CoreContainer container = new CoreContainer(); + + SolrResourceLoader loader = new SolrResourceLoader(createTempDir()); + Files.copy(TEST_PATH().resolve("solr.xml"), loader.getInstancePath().resolve("solr.xml")); + CoreContainer container = new CoreContainer(loader); container.load(); - container.create(new CoreDescriptor(container, "collection1", "collection1", "collection", "collection1")); + container.create("collection1_" + shard, ImmutableMap.of("collection", "collection1")); containerMap.put(port, container); System.clearProperty("solr.solr.home"); System.clearProperty("hostPort"); diff --git a/solr/core/src/test/org/apache/solr/core/TestConfigSets.java b/solr/core/src/test/org/apache/solr/core/TestConfigSets.java index ba226d6419f..6365f842a4e 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfigSets.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfigSets.java @@ -23,6 +23,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; +import com.google.common.collect.ImmutableMap; import org.apache.commons.io.FileUtils; import org.apache.solr.SolrTestCaseJ4; import org.junit.Rule; @@ -76,8 +77,7 @@ public class TestConfigSets extends SolrTestCaseJ4 { container = setupContainer(TEST_PATH().resolve("configsets").toString()); Path testDirectory = container.getResourceLoader().getInstancePath(); - SolrCore core1 = container.create(new CoreDescriptor(container, "core1", testDirectory.resolve("core1").toString(), - "configSet", "configset-2")); + SolrCore core1 = container.create("core1", ImmutableMap.of("configSet", "configset-2")); assertThat(core1.getCoreDescriptor().getName(), is("core1")); assertThat(Paths.get(core1.getDataDir()).toString(), is(testDirectory.resolve("core1").resolve("data").toString())); } @@ -94,7 +94,7 @@ public class TestConfigSets extends SolrTestCaseJ4 { container = setupContainer(getFile("solr/configsets").getAbsolutePath()); Path testDirectory = container.getResourceLoader().getInstancePath(); - container.create(new CoreDescriptor(container, "core1", testDirectory.resolve("core1").toString(), "configSet", "nonexistent")); + container.create("core1", ImmutableMap.of("configSet", "nonexistent")); fail("Expected core creation to fail"); } catch (Exception e) { @@ -122,12 +122,12 @@ public class TestConfigSets extends SolrTestCaseJ4 { CoreContainer container = new CoreContainer(SolrXmlConfig.fromString(loader, solrxml)); container.load(); - // We initially don't have a /get handler defined - SolrCore core = container.create(new CoreDescriptor(container, "core1", testDirectory + "/core", "configSet", "configset-2")); + // We initially don't have a /dump handler defined + SolrCore core = container.create("core1", ImmutableMap.of("configSet", "configset-2")); assertThat("No /dump handler should be defined in the initial configuration", core.getRequestHandler("/dump"), is(nullValue())); - // Now copy in a config with a /get handler and reload + // Now copy in a config with a /dump handler and reload FileUtils.copyFile(getFile("solr/collection1/conf/solrconfig-withgethandler.xml"), new File(new File(configSetsDir, "configset-2/conf"), "solrconfig.xml")); container.reload("core1"); diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java index 44c90a3246d..3cc5ab547ed 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java +++ b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java @@ -29,6 +29,8 @@ import java.util.jar.JarEntry; import java.util.jar.JarOutputStream; import java.util.regex.Pattern; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableMap; import org.apache.commons.io.FileUtils; import org.apache.lucene.util.IOUtils; import org.apache.solr.SolrTestCaseJ4; @@ -88,11 +90,8 @@ public class TestCoreContainer extends SolrTestCaseJ4 { CoreContainer cores = init(CONFIGSETS_SOLR_XML); try { - CoreDescriptor descriptor1 = new CoreDescriptor(cores, "core1", "./collection1", "configSet", "minimal"); - SolrCore core1 = cores.create(descriptor1); - - CoreDescriptor descriptor2 = new CoreDescriptor(cores, "core2", "./collection1", "configSet", "minimal"); - SolrCore core2 = cores.create(descriptor2); + SolrCore core1 = cores.create("core1", ImmutableMap.of("configSet", "minimal")); + SolrCore core2 = cores.create("core2", ImmutableMap.of("configSet", "minimal")); assertSame(core1.getLatestSchema(), core2.getLatestSchema()); @@ -105,9 +104,8 @@ public class TestCoreContainer extends SolrTestCaseJ4 { @Test public void testReloadSequential() throws Exception { final CoreContainer cc = init(CONFIGSETS_SOLR_XML); - CoreDescriptor descriptor1 = new CoreDescriptor(cc, "core1", "./collection1", "configSet", "minimal"); - cc.create(descriptor1); try { + cc.create("core1", ImmutableMap.of("configSet", "minimal")); cc.reload("core1"); cc.reload("core1"); cc.reload("core1"); @@ -121,8 +119,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 { @Test public void testReloadThreaded() throws Exception { final CoreContainer cc = init(CONFIGSETS_SOLR_XML); - CoreDescriptor descriptor1 = new CoreDescriptor(cc, "core1", "./collection1", "configSet", "minimal"); - cc.create(descriptor1); + cc.create("core1", ImmutableMap.of("configSet", "minimal")); class TestThread extends Thread { @Override @@ -159,8 +156,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 { assertEquals("There should not be cores", 0, cores.getCores().size()); //add a new core - CoreDescriptor coreDescriptor = new CoreDescriptor(cores, "core1", "collection1", CoreDescriptor.CORE_CONFIGSET, "minimal"); - SolrCore newCore = cores.create(coreDescriptor); + cores.create("core1", ImmutableMap.of("configSet", "minimal")); //assert one registered core @@ -218,7 +214,8 @@ public class TestCoreContainer extends SolrTestCaseJ4 { System.setProperty("configsets", getFile("solr/configsets").getAbsolutePath()); final CoreContainer cc = new CoreContainer(SolrXmlConfig.fromString(resourceLoader, CONFIGSETS_SOLR_XML), new Properties(), cl); - CoreDescriptor badcore = new CoreDescriptor(cc, "badcore", "badcore", "configSet", "nosuchconfigset"); + Path corePath = resourceLoader.getInstancePath().resolve("badcore"); + CoreDescriptor badcore = new CoreDescriptor(cc, "badcore", corePath, "configSet", "nosuchconfigset"); cl.add(badcore); try { @@ -229,7 +226,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 { assertThat(cc.getCoreInitFailures().size(), is(0)); // can we create the core now with a good config? - SolrCore core = cc.create(new CoreDescriptor(cc, "badcore", "badcore", "configSet", "minimal")); + SolrCore core = cc.create("badcore", ImmutableMap.of("configSet", "minimal")); assertThat(core, not(nullValue())); } @@ -246,8 +243,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 { ClassLoader contextLoader = Thread.currentThread().getContextClassLoader(); assertSame(contextLoader, sharedLoader.getParent()); - CoreDescriptor descriptor1 = new CoreDescriptor(cc, "core1", "./collection1", "configSet", "minimal"); - SolrCore core1 = cc.create(descriptor1); + SolrCore core1 = cc.create("core1", ImmutableMap.of("configSet", "minimal")); ClassLoader coreLoader = core1.getResourceLoader().getClassLoader(); assertSame(sharedLoader, coreLoader.getParent()); @@ -413,15 +409,15 @@ public class TestCoreContainer extends SolrTestCaseJ4 { assertEquals("wrong number of core failures", 0, failures.size()); // ----- - // try to add a collection with a path that doesn't exist - final CoreDescriptor bogus = new CoreDescriptor(cc, "bogus", "bogus_path"); + // try to add a collection with a configset that doesn't exist try { ignoreException(Pattern.quote("bogus_path")); - cc.create(bogus); + cc.create("bogus", ImmutableMap.of("configSet", "bogus_path")); fail("bogus inst dir failed to trigger exception from create"); } catch (SolrException e) { - assertTrue("init exception doesn't mention bogus dir: " + e.getCause().getCause().getMessage(), - 0 < e.getCause().getCause().getMessage().indexOf("bogus_path")); + Throwable cause = Throwables.getRootCause(e); + assertTrue("init exception doesn't mention bogus dir: " + cause.getMessage(), + 0 < cause.getMessage().indexOf("bogus_path")); } @@ -436,8 +432,8 @@ public class TestCoreContainer extends SolrTestCaseJ4 { assertEquals("wrong number of core failures", 1, failures.size()); fail = failures.get("bogus").exception; assertNotNull("null failure for test core", fail); - assertTrue("init failure doesn't mention problem: " + fail.getCause().getMessage(), - 0 < fail.getCause().getMessage().indexOf("bogus_path")); + assertTrue("init failure doesn't mention problem: " + fail.getMessage(), + 0 < fail.getMessage().indexOf("bogus_path")); // check that we get null accessing a non-existent core assertNull(cc.getCore("does_not_exist")); @@ -447,8 +443,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 { fail("Failed to get Exception on accessing core with init failure"); } catch (SolrException ex) { assertEquals(500, ex.code()); - // double wrapped - String cause = ex.getCause().getCause().getMessage(); + String cause = Throwables.getRootCause(ex).getMessage(); assertTrue("getCore() ex cause doesn't mention init fail: " + cause, 0 < cause.indexOf("bogus_path")); @@ -474,8 +469,8 @@ public class TestCoreContainer extends SolrTestCaseJ4 { System.setProperty("configsets", getFile("solr/configsets").getAbsolutePath()); final CoreContainer cc = new CoreContainer(SolrXmlConfig.fromString(resourceLoader, CONFIGSETS_SOLR_XML), new Properties(), cl); - cl.add(new CoreDescriptor(cc, "col_ok", "col_ok", "configSet", "minimal")); - cl.add(new CoreDescriptor(cc, "col_bad", "col_bad", "configSet", "bad-mergepolicy")); + cl.add(new CoreDescriptor(cc, "col_ok", resourceLoader.getInstancePath().resolve("col_ok"), "configSet", "minimal")); + cl.add(new CoreDescriptor(cc, "col_bad", resourceLoader.getInstancePath().resolve("col_bad"), "configSet", "bad-mergepolicy")); cc.load(); // check that we have the cores we expect @@ -513,8 +508,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 { FileUtils.getFile(cc.getSolrHome(), "col_bad", "conf", "solrconfig.xml")); FileUtils.copyFile(getFile("solr/collection1/conf/schema-minimal.xml"), FileUtils.getFile(cc.getSolrHome(), "col_bad", "conf", "schema.xml")); - final CoreDescriptor fixed = new CoreDescriptor(cc, "col_bad", "col_bad"); - cc.create(fixed); + cc.create("col_bad", ImmutableMap.of()); // check that we have the cores we expect cores = cc.getCoreNames(); @@ -531,10 +525,9 @@ public class TestCoreContainer extends SolrTestCaseJ4 { // ----- // try to add a collection with a path that doesn't exist - final CoreDescriptor bogus = new CoreDescriptor(cc, "bogus", "bogus_path"); try { ignoreException(Pattern.quote("bogus_path")); - cc.create(bogus); + cc.create("bogus", ImmutableMap.of("configSet", "bogus_path")); fail("bogus inst dir failed to trigger exception from create"); } catch (SolrException e) { assertTrue("init exception doesn't mention bogus dir: " + e.getCause().getCause().getMessage(), @@ -555,8 +548,8 @@ public class TestCoreContainer extends SolrTestCaseJ4 { assertEquals("wrong number of core failures", 1, failures.size()); fail = failures.get("bogus").exception; assertNotNull("null failure for test core", fail); - assertTrue("init failure doesn't mention problem: " + fail.getCause().getMessage(), - 0 < fail.getCause().getMessage().indexOf("bogus_path")); + assertTrue("init failure doesn't mention problem: " + fail.getMessage(), + 0 < fail.getMessage().indexOf("bogus_path")); // check that we get null accessing a non-existent core assertNull(cc.getCore("does_not_exist")); @@ -567,7 +560,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 { } catch (SolrException ex) { assertEquals(500, ex.code()); // double wrapped - String cause = ex.getCause().getCause().getMessage(); + String cause = ex.getCause().getMessage(); assertTrue("getCore() ex cause doesn't mention init fail: " + cause, 0 < cause.indexOf("bogus_path")); } diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java b/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java index 971e9860163..286a53dc13f 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java +++ b/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java @@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.Properties; +import com.google.common.collect.ImmutableMap; import org.apache.commons.io.FileUtils; import org.apache.lucene.util.IOUtils; import org.apache.solr.SolrTestCaseJ4; @@ -68,7 +69,6 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 { props.put(CoreDescriptor.CORE_TRANSIENT, Boolean.toString(isLazy)); props.put(CoreDescriptor.CORE_LOADONSTARTUP, Boolean.toString(loadOnStartup)); props.put(CoreDescriptor.CORE_DATADIR, "${core.dataDir:stuffandnonsense}"); - props.put(CoreDescriptor.CORE_INSTDIR, "totallybogus"); // For testing that this property is ignored if present. for (String extra : extraProps) { String[] parts = extra.split("="); @@ -155,9 +155,6 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 { // This is too long and ugly to put in. Besides, it varies. assertNotNull(desc.getInstanceDir()); - // Prove we're ignoring this even though it's set in the properties file - assertFalse("InstanceDir should be ignored", desc.getInstanceDir().contains("totallybogus")); - assertEquals("core1", desc.getDataDir()); assertEquals("solrconfig-minimal.xml", desc.getConfigName()); assertEquals("schema-tiny.xml", desc.getSchemaName()); @@ -302,8 +299,8 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 { assertNull(cc.getCore("core0")); - SolrCore core3 = cc.create(new CoreDescriptor(cc, "core3", "core3", "configSet", "minimal")); - assertThat(core3.getCoreDescriptor().getInstanceDir(), containsString("relative")); + SolrCore core3 = cc.create("core3", ImmutableMap.of("configSet", "minimal")); + assertThat(core3.getCoreDescriptor().getInstanceDir().toAbsolutePath().toString(), containsString("relative")); } finally { cc.shutdown(); diff --git a/solr/core/src/test/org/apache/solr/core/TestLazyCores.java b/solr/core/src/test/org/apache/solr/core/TestLazyCores.java index 2f15644865a..5c481adee5d 100644 --- a/solr/core/src/test/org/apache/solr/core/TestLazyCores.java +++ b/solr/core/src/test/org/apache/solr/core/TestLazyCores.java @@ -49,8 +49,8 @@ public class TestLazyCores extends SolrTestCaseJ4 { private File solrHomeDirectory; - private static CoreDescriptor makeCoreDescriptor(CoreContainer cc, String coreName, String instanceDir, String isTransient, String loadOnStartup) { - return new CoreDescriptor(cc, coreName, instanceDir, + private static CoreDescriptor makeCoreDescriptor(CoreContainer cc, String coreName, String isTransient, String loadOnStartup) { + return new CoreDescriptor(cc, coreName, cc.getCoreRootDirectory().resolve(coreName), CoreDescriptor.CORE_TRANSIENT, isTransient, CoreDescriptor.CORE_LOADONSTARTUP, loadOnStartup); } @@ -59,15 +59,15 @@ public class TestLazyCores extends SolrTestCaseJ4 { @Override public List discover(CoreContainer cc) { return ImmutableList.of( - new CoreDescriptor(cc, "collection1", "collection1"), - makeCoreDescriptor(cc, "collectionLazy2", "collection2", "true", "true"), - makeCoreDescriptor(cc, "collectionLazy3", "collection3", "on", "false"), - makeCoreDescriptor(cc, "collectionLazy4", "collection4", "false", "false"), - makeCoreDescriptor(cc, "collectionLazy5", "collection5", "false", "true"), - makeCoreDescriptor(cc, "collectionLazy6", "collection6", "true", "false"), - makeCoreDescriptor(cc, "collectionLazy7", "collection7", "true", "false"), - makeCoreDescriptor(cc, "collectionLazy8", "collection8", "true", "false"), - makeCoreDescriptor(cc, "collectionLazy9", "collection9", "true", "false") + makeCoreDescriptor(cc, "collection1", "false", "true"), + makeCoreDescriptor(cc, "collection2", "true", "true"), + makeCoreDescriptor(cc, "collection3", "on", "false"), + makeCoreDescriptor(cc, "collection4", "false", "false"), + makeCoreDescriptor(cc, "collection5", "false", "true"), + makeCoreDescriptor(cc, "collection6", "true", "false"), + makeCoreDescriptor(cc, "collection7", "true", "false"), + makeCoreDescriptor(cc, "collection8", "true", "false"), + makeCoreDescriptor(cc, "collection9", "true", "false") ); } }; @@ -91,28 +91,28 @@ public class TestLazyCores extends SolrTestCaseJ4 { try { // NOTE: This checks the initial state for loading, no need to do this elsewhere. - checkInCores(cc, "collection1", "collectionLazy2", "collectionLazy5"); - checkNotInCores(cc, "collectionLazy3", "collectionLazy4", "collectionLazy6", "collectionLazy7", - "collectionLazy8", "collectionLazy9"); + checkInCores(cc, "collection1", "collection2", "collection5"); + checkNotInCores(cc, "collection3", "collection4", "collection6", "collection7", + "collection8", "collection9"); SolrCore core1 = cc.getCore("collection1"); assertFalse("core1 should not be transient", core1.getCoreDescriptor().isTransient()); assertTrue("core1 should be loadable", core1.getCoreDescriptor().isLoadOnStartup()); assertNotNull(core1.getSolrConfig()); - SolrCore core2 = cc.getCore("collectionLazy2"); + SolrCore core2 = cc.getCore("collection2"); assertTrue("core2 should be transient", core2.getCoreDescriptor().isTransient()); assertTrue("core2 should be loadable", core2.getCoreDescriptor().isLoadOnStartup()); - SolrCore core3 = cc.getCore("collectionLazy3"); + SolrCore core3 = cc.getCore("collection3"); assertTrue("core3 should be transient", core3.getCoreDescriptor().isTransient()); assertFalse("core3 should not be loadable", core3.getCoreDescriptor().isLoadOnStartup()); - SolrCore core4 = cc.getCore("collectionLazy4"); + SolrCore core4 = cc.getCore("collection4"); assertFalse("core4 should not be transient", core4.getCoreDescriptor().isTransient()); assertFalse("core4 should not be loadable", core4.getCoreDescriptor().isLoadOnStartup()); - SolrCore core5 = cc.getCore("collectionLazy5"); + SolrCore core5 = cc.getCore("collection5"); assertFalse("core5 should not be transient", core5.getCoreDescriptor().isTransient()); assertTrue("core5 should be loadable", core5.getCoreDescriptor().isLoadOnStartup()); @@ -168,8 +168,8 @@ public class TestLazyCores extends SolrTestCaseJ4 { CoreContainer cc = init(); try { // Make sure Lazy4 isn't loaded. Should be loaded on the get - checkNotInCores(cc, "collectionLazy4"); - SolrCore core4 = cc.getCore("collectionLazy4"); + checkNotInCores(cc, "collection4"); + SolrCore core4 = cc.getCore("collection4"); checkSearch(core4); @@ -181,7 +181,7 @@ public class TestLazyCores extends SolrTestCaseJ4 { , "//result[@numFound='0']" ); - checkInCores(cc, "collectionLazy4"); + checkInCores(cc, "collection4"); core4.close(); collection1.close(); @@ -196,40 +196,39 @@ public class TestLazyCores extends SolrTestCaseJ4 { try { // First check that all the cores that should be loaded at startup actually are. - checkInCores(cc, "collection1", "collectionLazy2", "collectionLazy5"); - checkNotInCores(cc, "collectionLazy3", "collectionLazy4", "collectionLazy6", - "collectionLazy7", "collectionLazy8", "collectionLazy9"); + checkInCores(cc, "collection1", "collection2", "collection5"); + checkNotInCores(cc, "collection3", "collection4", "collection6", "collection7", "collection8", "collection9"); // By putting these in non-alpha order, we're also checking that we're not just seeing an artifact. SolrCore core1 = cc.getCore("collection1"); - SolrCore core3 = cc.getCore("collectionLazy3"); - SolrCore core4 = cc.getCore("collectionLazy4"); - SolrCore core2 = cc.getCore("collectionLazy2"); - SolrCore core5 = cc.getCore("collectionLazy5"); + SolrCore core3 = cc.getCore("collection3"); + SolrCore core4 = cc.getCore("collection4"); + SolrCore core2 = cc.getCore("collection2"); + SolrCore core5 = cc.getCore("collection5"); - checkInCores(cc, "collection1", "collectionLazy2", "collectionLazy3", "collectionLazy4", "collectionLazy5"); - checkNotInCores(cc, "collectionLazy6", "collectionLazy7", "collectionLazy8", "collectionLazy9"); + checkInCores(cc, "collection1", "collection2", "collection3", "collection4", "collection5"); + checkNotInCores(cc, "collection6", "collection7", "collection8", "collection9"); // map should be full up, add one more and verify - SolrCore core6 = cc.getCore("collectionLazy6"); - checkInCores(cc, "collection1", "collectionLazy2", "collectionLazy3", "collectionLazy4", "collectionLazy5", - "collectionLazy6"); - checkNotInCores(cc, "collectionLazy7", "collectionLazy8", "collectionLazy9"); + SolrCore core6 = cc.getCore("collection6"); + checkInCores(cc, "collection1", "collection2", "collection3", "collection4", "collection5", + "collection6"); + checkNotInCores(cc, "collection7", "collection8", "collection9"); - SolrCore core7 = cc.getCore("collectionLazy7"); - checkInCores(cc, "collection1", "collectionLazy2", "collectionLazy3", "collectionLazy4", "collectionLazy5", - "collectionLazy6", "collectionLazy7"); - checkNotInCores(cc, "collectionLazy8", "collectionLazy9"); + SolrCore core7 = cc.getCore("collection7"); + checkInCores(cc, "collection1", "collection2", "collection3", "collection4", "collection5", + "collection6", "collection7"); + checkNotInCores(cc, "collection8", "collection9"); - SolrCore core8 = cc.getCore("collectionLazy8"); - checkInCores(cc, "collection1", "collectionLazy2", "collectionLazy4", "collectionLazy5", "collectionLazy6", - "collectionLazy7", "collectionLazy8"); - checkNotInCores(cc, "collectionLazy3", "collectionLazy9"); + SolrCore core8 = cc.getCore("collection8"); + checkInCores(cc, "collection1", "collection2", "collection4", "collection5", "collection6", + "collection7", "collection8"); + checkNotInCores(cc, "collection3", "collection9"); - SolrCore core9 = cc.getCore("collectionLazy9"); - checkInCores(cc, "collection1", "collectionLazy4", "collectionLazy5", "collectionLazy6", "collectionLazy7", - "collectionLazy8", "collectionLazy9"); - checkNotInCores(cc, "collectionLazy2", "collectionLazy3"); + SolrCore core9 = cc.getCore("collection9"); + checkInCores(cc, "collection1", "collection4", "collection5", "collection6", "collection7", + "collection8", "collection9"); + checkNotInCores(cc, "collection2", "collection3"); // Note decrementing the count when the core is removed from the lazyCores list is appropriate, since the @@ -261,7 +260,7 @@ public class TestLazyCores extends SolrTestCaseJ4 { threads[idx] = new Thread() { @Override public void run() { - SolrCore core = cc.getCore("collectionLazy3"); + SolrCore core = cc.getCore("collection3"); synchronized (theCores) { theCores.add(core); } @@ -312,10 +311,10 @@ public class TestLazyCores extends SolrTestCaseJ4 { try { // First, try all 4 combinations of load on startup and transient final CoreAdminHandler admin = new CoreAdminHandler(cc); - SolrCore lc2 = cc.getCore("collectionLazy2"); - SolrCore lc4 = cc.getCore("collectionLazy4"); - SolrCore lc5 = cc.getCore("collectionLazy5"); - SolrCore lc6 = cc.getCore("collectionLazy6"); + SolrCore lc2 = cc.getCore("collection2"); + SolrCore lc4 = cc.getCore("collection4"); + SolrCore lc5 = cc.getCore("collection5"); + SolrCore lc6 = cc.getCore("collection6"); copyMinConf(new File(solrHomeDirectory, "t2")); copyMinConf(new File(solrHomeDirectory, "t4")); @@ -324,10 +323,10 @@ public class TestLazyCores extends SolrTestCaseJ4 { // Should also fail with the same name - tryCreateFail(admin, "collectionLazy2", "t12", "Core with name", "collectionLazy2", "already exists"); - tryCreateFail(admin, "collectionLazy4", "t14", "Core with name", "collectionLazy4", "already exists"); - tryCreateFail(admin, "collectionLazy5", "t15", "Core with name", "collectionLazy5", "already exists"); - tryCreateFail(admin, "collectionLazy6", "t16", "Core with name", "collectionLazy6", "already exists"); + tryCreateFail(admin, "collection2", "t12", "Core with name", "collection2", "already exists"); + tryCreateFail(admin, "collection4", "t14", "Core with name", "collection4", "already exists"); + tryCreateFail(admin, "collection5", "t15", "Core with name", "collection5", "already exists"); + tryCreateFail(admin, "collection6", "t16", "Core with name", "collection6", "already exists"); lc2.close(); lc4.close(); @@ -339,7 +338,7 @@ public class TestLazyCores extends SolrTestCaseJ4 { } } - private void createViaAdmin(CoreContainer cc, String name, String instanceDir, boolean isTransient, + private void createViaAdmin(CoreContainer cc, String name, boolean isTransient, boolean loadOnStartup) throws Exception { final CoreAdminHandler admin = new CoreAdminHandler(cc); @@ -347,7 +346,6 @@ public class TestLazyCores extends SolrTestCaseJ4 { admin.handleRequestBody (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATE.toString(), - CoreAdminParams.INSTANCE_DIR, instanceDir, CoreAdminParams.NAME, name, CoreAdminParams.TRANSIENT, Boolean.toString(isTransient), CoreAdminParams.LOAD_ON_STARTUP, Boolean.toString(loadOnStartup)), @@ -379,11 +377,11 @@ public class TestLazyCores extends SolrTestCaseJ4 { copyMinConf(new File(solrHomeDirectory, "core4")); copyMinConf(new File(solrHomeDirectory, "core5")); - createViaAdmin(cc, "core1", "./core1", true, true); - createViaAdmin(cc, "core2", "./core2", true, false); - createViaAdmin(cc, "core3", "./core3", true, true); - createViaAdmin(cc, "core4", "./core4", true, false); - createViaAdmin(cc, "core5", "./core5", true, false); + createViaAdmin(cc, "core1", true, true); + createViaAdmin(cc, "core2", true, false); + createViaAdmin(cc, "core3", true, true); + createViaAdmin(cc, "core4", true, false); + createViaAdmin(cc, "core5", true, false); SolrCore c1 = cc.getCore("core1"); SolrCore c2 = cc.getCore("core2"); @@ -391,10 +389,10 @@ public class TestLazyCores extends SolrTestCaseJ4 { SolrCore c4 = cc.getCore("core4"); SolrCore c5 = cc.getCore("core5"); - checkNotInCores(cc, "core1", "collectionLazy2", "collectionLazy3", "collectionLazy4", "collectionLazy6" - , "collectionLazy7", "collectionLazy8", "collectionLazy9"); + checkNotInCores(cc, "core1", "collection2", "collection3", "collection4", "collection6" + , "collection7", "collection8", "collection9"); - checkInCores(cc, "collection1", "collectionLazy5", "core2", "core3", "core4", "core5"); + checkInCores(cc, "collection1", "collection5", "core2", "core3", "core4", "core5"); // While we're at it, a test for SOLR-5366, unloading transient core that's been unloaded b/c it's // transient generates a "too many closes" errorl diff --git a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java index 29aba01f4e7..2149267a142 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerTest.java @@ -17,6 +17,11 @@ package org.apache.solr.handler.admin; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; import org.apache.commons.codec.Charsets; import org.apache.commons.io.FileUtils; @@ -38,9 +43,6 @@ import org.junit.Test; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; -import java.io.File; -import java.util.Map; - public class CoreAdminHandlerTest extends SolrTestCaseJ4 { @BeforeClass @@ -117,16 +119,15 @@ public class CoreAdminHandlerTest extends SolrTestCaseJ4 { final CoreAdminHandler admin = new CoreAdminHandler(cores); - String instDir; + Path instDir; try (SolrCore template = cores.getCore("collection1")) { assertNotNull(template); instDir = template.getCoreDescriptor().getInstanceDir(); } - - final File instDirFile = new File(instDir); - assertTrue("instDir doesn't exist: " + instDir, instDirFile.exists()); + + assertTrue("instDir doesn't exist: " + instDir, Files.exists(instDir)); final File instPropFile = new File(workDir, "instProp"); - FileUtils.copyDirectory(instDirFile, instPropFile); + FileUtils.copyDirectory(instDir.toFile(), instPropFile); // create a new core (using CoreAdminHandler) w/ properties diff --git a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminRequestStatusTest.java b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminRequestStatusTest.java index a647eb36591..70df29eac73 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminRequestStatusTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminRequestStatusTest.java @@ -18,6 +18,8 @@ package org.apache.solr.handler.admin; */ import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; import org.apache.commons.io.FileUtils; import org.apache.solr.SolrTestCaseJ4; @@ -44,16 +46,15 @@ public class CoreAdminRequestStatusTest extends SolrTestCaseJ4{ final CoreAdminHandler admin = new CoreAdminHandler(cores); - String instDir; + Path instDir; try (SolrCore template = cores.getCore("collection1")) { assertNotNull(template); instDir = template.getCoreDescriptor().getInstanceDir(); } - final File instDirFile = new File(instDir); - assertTrue("instDir doesn't exist: " + instDir, instDirFile.exists()); + assertTrue("instDir doesn't exist: " + instDir, Files.exists(instDir)); final File instPropFile = new File(workDir, "instProp"); - FileUtils.copyDirectory(instDirFile, instPropFile); + FileUtils.copyDirectory(instDir.toFile(), instPropFile); // create a new core (using CoreAdminHandler) w/ properties diff --git a/solr/core/src/test/org/apache/solr/handler/component/SuggestComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/SuggestComponentTest.java index 89eba5aff26..1ba2e8281f3 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/SuggestComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/SuggestComponentTest.java @@ -22,16 +22,16 @@ import org.apache.solr.common.SolrException; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.SolrCore; -import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.spelling.suggest.SuggesterParams; -import org.apache.solr.util.RefCounted; import org.junit.BeforeClass; import org.junit.Test; public class SuggestComponentTest extends SolrTestCaseJ4 { - static String rh = "/suggest"; + private static final String rh = "/suggest"; + + private static CoreContainer cc; @BeforeClass public static void beforeClass() throws Exception { @@ -506,8 +506,9 @@ public class SuggestComponentTest extends SolrTestCaseJ4 { SolrCore core = h.getCore(); String dataDir1 = core.getDataDir(); CoreDescriptor cd = core.getCoreDescriptor(); - cores.unload(core.getName()); - SolrCore createdCore = cores.create(cd); + h.close(); + createCore(); + SolrCore createdCore = h.getCore(); assertEquals(dataDir1, createdCore.getDataDir()); assertEquals(createdCore, h.getCore()); } else { diff --git a/solr/core/src/test/org/apache/solr/search/TestIndexSearcher.java b/solr/core/src/test/org/apache/solr/search/TestIndexSearcher.java index 5eb2bd74479..e41635c4405 100644 --- a/solr/core/src/test/org/apache/solr/search/TestIndexSearcher.java +++ b/solr/core/src/test/org/apache/solr/search/TestIndexSearcher.java @@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.collect.ImmutableMap; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; @@ -219,9 +220,8 @@ public class TestIndexSearcher extends SolrTestCaseJ4 { MockSearcherListener.numberOfTimesCalledFirstSearcher = new AtomicInteger(); try { - CoreDescriptor newCd = new CoreDescriptor(cores, "core1", cd.getInstanceDir(), "config", "solrconfig-searcher-listeners1.xml"); // Create a new core, this should call all the firstSearcherListeners - newCore = cores.create(newCd); + newCore = cores.create("core1", cd.getInstanceDir(), ImmutableMap.of("config", "solrconfig-searcher-listeners1.xml")); //validate that the new core was created with the correct solrconfig assertNotNull(newCore.getSearchComponent("mock")); @@ -270,9 +270,8 @@ public class TestIndexSearcher extends SolrTestCaseJ4 { final SolrCore newCore; boolean coreCreated = false; try { - CoreDescriptor newCd = new CoreDescriptor(cores, "core1", cd.getInstanceDir(), "config", "solrconfig-searcher-listeners1.xml"); // Create a new core, this should call all the firstSearcherListeners - newCore = cores.create(newCd); + newCore = cores.create("core1", cd.getInstanceDir(), ImmutableMap.of("config", "solrconfig-searcher-listeners1.xml")); coreCreated = true; //validate that the new core was created with the correct solrconfig @@ -337,9 +336,8 @@ public class TestIndexSearcher extends SolrTestCaseJ4 { boolean coreCreated = false; try { System.setProperty("tests.solr.useColdSearcher", "true"); - CoreDescriptor newCd = new CoreDescriptor(cores, "core1", cd.getInstanceDir(), "config", "solrconfig-searcher-listeners1.xml"); // Create a new core, this should call all the firstSearcherListeners - newCore = cores.create(newCd); + newCore = cores.create("core1", cd.getInstanceDir(), ImmutableMap.of("config", "solrconfig-searcher-listeners1.xml")); coreCreated = true; //validate that the new core was created with the correct solrconfig diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java index f128f9ce2e4..eb7c0216492 100644 --- a/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java +++ b/solr/core/src/test/org/apache/solr/update/SolrIndexSplitterTest.java @@ -17,6 +17,12 @@ package org.apache.solr.update; * limitations under the License. */ +import java.io.File; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.Term; @@ -28,7 +34,6 @@ import org.apache.solr.common.cloud.CompositeIdRouter; import org.apache.solr.common.cloud.DocRouter; import org.apache.solr.common.cloud.PlainIdRouter; import org.apache.solr.common.util.Hash; -import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.SolrCore; import org.apache.solr.request.LocalSolrQueryRequest; @@ -36,11 +41,6 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import java.io.File; -import java.io.UnsupportedEncodingException; -import java.nio.charset.StandardCharsets; -import java.util.List; - public class SolrIndexSplitterTest extends SolrTestCaseJ4 { File indexDir1 = null, indexDir2 = null, indexDir3 = null; @@ -159,15 +159,11 @@ public class SolrIndexSplitterTest extends SolrTestCaseJ4 { SolrCore core1 = null, core2 = null; try { - String instanceDir = h.getCore().getCoreDescriptor().getInstanceDir(); - CoreDescriptor dcore1 = buildCoreDescriptor(h.getCoreContainer(), "split1", instanceDir) - .withDataDir(indexDir1.getAbsolutePath()).withSchema("schema12.xml").build(); - core1 = h.getCoreContainer().create(dcore1); - - CoreDescriptor dcore2 = buildCoreDescriptor(h.getCoreContainer(), "split2", instanceDir) - .withDataDir(indexDir2.getAbsolutePath()).withSchema("schema12.xml").build(); - core2 = h.getCoreContainer().create(dcore2); + core1 = h.getCoreContainer().create("split1", + ImmutableMap.of("dataDir", indexDir1.getAbsolutePath(), "configSet", "minimal")); + core2 = h.getCoreContainer().create("split2", + ImmutableMap.of("dataDir", indexDir2.getAbsolutePath(), "configSet", "minimal")); LocalSolrQueryRequest request = null; try { diff --git a/solr/core/src/test/org/apache/solr/util/MockCoreContainer.java b/solr/core/src/test/org/apache/solr/util/MockCoreContainer.java index 3d736df2a5c..314cdc47804 100644 --- a/solr/core/src/test/org/apache/solr/util/MockCoreContainer.java +++ b/solr/core/src/test/org/apache/solr/util/MockCoreContainer.java @@ -17,6 +17,9 @@ package org.apache.solr.util; * limitations under the License. */ +import java.nio.file.Path; +import java.nio.file.Paths; + import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; @@ -24,7 +27,7 @@ import org.apache.solr.core.CoreDescriptor; public class MockCoreContainer extends CoreContainer { public static class MockCoreDescriptor extends CoreDescriptor { public MockCoreDescriptor() { - super(new MockCoreContainer(), "mock", "path"); + super(new MockCoreContainer(), "mock", Paths.get("path")); } } @@ -32,7 +35,7 @@ public class MockCoreContainer extends CoreContainer { super(new Object()); } - public String getCoreRootDirectory() { - return "coreroot"; + public Path getCoreRootDirectory() { + return Paths.get("coreroot"); } } diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index ca65fd559a3..37f77e8a4dd 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -71,7 +71,6 @@ import org.apache.solr.common.util.ObjectReleaseTracker; import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.common.util.XML; import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.CoresLocator; import org.apache.solr.core.NodeConfig; import org.apache.solr.core.SolrConfig; @@ -397,9 +396,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { configString = config; schemaString = schema; testSolrHome = Paths.get(solrHome); - if (solrHome != null) { - System.setProperty("solr.solr.home", solrHome); - } + System.setProperty("solr.solr.home", solrHome); initCore(); } @@ -1862,53 +1859,6 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { FileUtils.copyFile(new File(top, "synonyms.txt"), new File(subHome, "synonyms.txt")); } - public static CoreDescriptorBuilder buildCoreDescriptor(CoreContainer container, String name, String instancedir) { - return new CoreDescriptorBuilder(container, name, instancedir); - } - - public static class CoreDescriptorBuilder { - - final String name; - final String instanceDir; - final CoreContainer container; - final Properties properties = new Properties(); - - public CoreDescriptorBuilder(CoreContainer container, String name, String instancedir) { - this.name = name; - this.instanceDir = instancedir; - this.container = container; - } - - public CoreDescriptorBuilder withSchema(String schema) { - properties.setProperty(CoreDescriptor.CORE_SCHEMA, schema); - return this; - } - - public CoreDescriptorBuilder withConfig(String config) { - properties.setProperty(CoreDescriptor.CORE_CONFIG, config); - return this; - } - - public CoreDescriptorBuilder withDataDir(String datadir) { - properties.setProperty(CoreDescriptor.CORE_DATADIR, datadir); - return this; - } - - public CoreDescriptor build() { - return new CoreDescriptor(container, name, instanceDir, properties); - } - - public CoreDescriptorBuilder isTransient(boolean isTransient) { - properties.setProperty(CoreDescriptor.CORE_TRANSIENT, Boolean.toString(isTransient)); - return this; - } - - public CoreDescriptorBuilder loadOnStartup(boolean loadOnStartup) { - properties.setProperty(CoreDescriptor.CORE_LOADONSTARTUP, Boolean.toString(loadOnStartup)); - return this; - } - } - public boolean compareSolrDocument(Object expected, Object actual) { if (!(expected instanceof SolrDocument) || !(actual instanceof SolrDocument)) { diff --git a/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java b/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java index 05a3af8ecbe..444f02a7479 100644 --- a/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java +++ b/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java @@ -202,7 +202,7 @@ public class TestHarness extends BaseTestHarness { @Override public List discover(CoreContainer cc) { - return ImmutableList.of(new CoreDescriptor(cc, coreName, coreName, + return ImmutableList.of(new CoreDescriptor(cc, coreName, cc.getCoreRootDirectory().resolve(coreName), CoreDescriptor.CORE_DATADIR, dataDir, CoreDescriptor.CORE_CONFIG, solrConfig, CoreDescriptor.CORE_SCHEMA, schema,