Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-5306: Extra collection creation parameters like collection.configName are not being respected. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1530772 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2013-10-09 19:46:00 +00:00
parent 52e63ba951
commit 8281cbf8af
7 changed files with 165 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -118,6 +118,9 @@ Bug Fixes
* SOLR-5314: Shard split action should use soft commits instead of hard commits
to make sub shard data visible. (Kalle Aaltonen, shalin)
* SOLR-5306: Extra collection creation parameters like collection.configName are
not being respected. (Mark Miller, Liang Tianyu, Nathan Neulinger)
Security
----------------------

18
solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
View File

@ -19,9 +19,11 @@ package org.apache.solr.core;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.util.IOUtils;
import org.apache.solr.util.PropertiesUtil;
@ -132,6 +134,19 @@ public class CoreDescriptor {
*/
public CoreDescriptor(CoreContainer container, String name, String instanceDir,
Properties coreProps) {
this(container, name, instanceDir, coreProps, null);
}
/**
* Create a new CoreDescriptor.
* @param container the CoreDescriptor's container
* @param name the CoreDescriptor's name
* @param instanceDir a String containing the instanceDir
* @param coreProps a Properties object of the properties for this core
* @param params additional params
*/
public CoreDescriptor(CoreContainer container, String name, String instanceDir,
Properties coreProps, SolrParams params) {
this.coreContainer = container;
@ -169,6 +184,9 @@ public class CoreDescriptor {
// TODO maybe make this a CloudCoreDescriptor subclass?
if (container.isZooKeeperAware()) {
cloudDesc = new CloudDescriptor(name, coreProperties);
if (params != null) {
cloudDesc.setParams(params);
}
}
else {
cloudDesc = null;

2
solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
View File

@ -455,7 +455,7 @@ public class CoreAdminHandler extends RequestHandlerBase {
coreProps.setProperty(propName, propValue);
}
return new CoreDescriptor(container, name, instancedir, coreProps);
return new CoreDescriptor(container, name, instancedir, coreProps, params);
}
private static String checkNotEmpty(String value, String message) {

107
solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
View File

@ -65,6 +65,7 @@ import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@ -99,6 +100,7 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
CompletionService<Request> completionService;
Set<Future<Request>> pending;
private boolean secondConfigSet = random().nextBoolean();
@BeforeClass
public static void beforeThisClass2() throws Exception {
@ -109,6 +111,34 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
@Override
public void setUp() throws Exception {
super.setUp();
if (secondConfigSet ) {
String zkHost = zkServer.getZkHost();
String zkAddress = zkServer.getZkAddress();
SolrZkClient zkClient = new SolrZkClient(zkHost, AbstractZkTestCase.TIMEOUT);
zkClient.makePath("/solr", false, true);
zkClient.close();
zkClient = new SolrZkClient(zkAddress, AbstractZkTestCase.TIMEOUT);
File solrhome = new File(TEST_HOME());
// for now, always upload the config and schema to the canonical names
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "solrconfig.xml", "solrconfig.xml");
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "schema.xml", "schema.xml");
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "solrconfig.snippet.randomindexconfig.xml");
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "stopwords.txt");
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "protwords.txt");
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "currency.xml");
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "open-exchange-rates.json");
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "mapping-ISOLatin1Accent.txt");
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "old_synonyms.txt");
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "synonyms.txt");
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "elevate.xml");
zkClient.close();
}
System.setProperty("numShards", Integer.toString(sliceCount));
System.setProperty("solr.xml.persist", "true");
}
@ -180,7 +210,6 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
params.set("name", collectionName);
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
try {
NamedList<Object> resp = createNewSolrServer("", baseUrl)
.request(request);
@ -205,6 +234,9 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
+ System.currentTimeMillis() + "halfcollection" + "_hdn";
createCmd.setDataDir(dataDir);
createCmd.setNumShards(2);
if (secondConfigSet) {
createCmd.setCollectionConfigName("conf1");
}
createNewSolrServer("", baseUrl).request(createCmd);
ModifiableSolrParams params = new ModifiableSolrParams();
@ -224,6 +256,9 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
params.set("numShards", 2);
request = new QueryRequest(params);
request.setPath("/admin/collections");
if (secondConfigSet) {
params.set("collection.configName", "conf1");
}
resp = createNewSolrServer("", baseUrl).request(request);
}
@ -231,8 +266,13 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
private void deleteCollectionWithDownNodes() throws Exception {
String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
// now try to remove a collection when a couple of it's nodes are down
createCollection(null, "halfdeletedcollection2", 3, 2, 6,
createNewSolrServer("", baseUrl), null);
if (secondConfigSet) {
createCollection(null, "halfdeletedcollection2", 3, 2, 6,
createNewSolrServer("", baseUrl), null, "conf2");
} else {
createCollection(null, "halfdeletedcollection2", 3, 2, 6,
createNewSolrServer("", baseUrl), null);
}
waitForRecoveriesToFinish("halfdeletedcollection2", false);
@ -286,6 +326,9 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
collectionName = "collection";
// No Name
// params.set("name", collectionName);
if (secondConfigSet) {
params.set("collection.configName", "conf1");
}
request = new QueryRequest(params);
request.setPath("/admin/collections");
gotExp = false;
@ -303,6 +346,9 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
collectionName = "collection";
params.set("name", collectionName);
params.set("numShards", 2);
if (secondConfigSet) {
params.set("collection.configName", "conf1");
}
params.set(REPLICATION_FACTOR, 10);
request = new QueryRequest(params);
request.setPath("/admin/collections");
@ -321,6 +367,9 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
collectionName = "acollection";
params.set("name", collectionName);
params.set(REPLICATION_FACTOR, 10);
if (secondConfigSet) {
params.set("collection.configName", "conf1");
}
request = new QueryRequest(params);
request.setPath("/admin/collections");
gotExp = false;
@ -339,6 +388,9 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
params.set("name", collectionName);
params.set(REPLICATION_FACTOR, 10);
params.set("numShards", 0);
if (secondConfigSet) {
params.set("collection.configName", "conf1");
}
request = new QueryRequest(params);
request.setPath("/admin/collections");
gotExp = false;
@ -361,6 +413,9 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
+ System.currentTimeMillis() + "halfcollection" + "_3n";
createCmd.setDataDir(dataDir);
createCmd.setNumShards(1);
if (secondConfigSet) {
createCmd.setCollectionConfigName("conf1");
}
createNewSolrServer("", baseUrl).request(createCmd);
createCmd = new Create();
@ -370,6 +425,9 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
+ System.currentTimeMillis() + "halfcollection" + "_3n";
createCmd.setDataDir(dataDir);
createCmd.setNumShards(1);
if (secondConfigSet) {
createCmd.setCollectionConfigName("conf1");
}
createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(1))).request(createCmd);
params = new ModifiableSolrParams();
@ -379,6 +437,10 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
params.set("numShards", 2);
params.set("wt", "xml");
if (secondConfigSet) {
params.set("collection.configName", "conf1");
}
String nn1 = ((SolrDispatchFilter) jettys.get(0).getDispatchFilter().getFilter()).getCores().getZkController().getNodeName();
String nn2 = ((SolrDispatchFilter) jettys.get(1).getDispatchFilter().getFilter()).getCores().getZkController().getNodeName();
@ -391,6 +453,9 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
SimpleOrderedMap success = (SimpleOrderedMap) resp.get("success");
SimpleOrderedMap failure = (SimpleOrderedMap) resp.get("failure");
assertNotNull(resp.toString(), success);
assertNotNull(resp.toString(), failure);
String val1 = success.getVal(0).toString();
String val2 = failure.getVal(0).toString();
assertTrue(val1.contains("SolrException") || val2.contains("SolrException"));
@ -408,6 +473,11 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
String collectionName = "nodes_used_collection";
params.set("name", collectionName);
if (secondConfigSet) {
params.set("collection.configName", "conf1");
}
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
createNewSolrServer("", baseUrl).request(request);
@ -477,9 +547,13 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
// and use this CloudSolrServer to create the collection
client = createCloudClient("awholynewcollection_" + i);
}
createCollection(collectionInfos, "awholynewcollection_" + i,
numShards, replicationFactor, maxShardsPerNode, client, null);
if (secondConfigSet) {
createCollection(collectionInfos, "awholynewcollection_" + i,
numShards, replicationFactor, maxShardsPerNode, client, null, "conf2");
} else {
createCollection(collectionInfos, "awholynewcollection_" + i,
numShards, replicationFactor, maxShardsPerNode, client, null);
}
} finally {
if (client != null) client.shutdown();
}
@ -501,6 +575,19 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
for (int j = 0; j < cnt; j++) {
waitForRecoveriesToFinish("awholynewcollection_" + j, zkStateReader, false);
if (secondConfigSet) {
// let's see if they are using the second config set
byte[] data = zkStateReader.getZkClient()
.getData(
ZkStateReader.COLLECTIONS_ZKNODE + "/" + "awholynewcollection_"
+ j, null, null, true);
assertNotNull(data);
ZkNodeProps props = ZkNodeProps.load(data);
String configName = props.getStr(ZkController.CONFIGNAME_PROP);
assertEquals("conf2", configName);
}
}
checkInstanceDirs(jettys.get(0));
@ -594,6 +681,9 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
collectionName = "acollectionafterbaddelete";
params.set("name", collectionName);
if (secondConfigSet) {
params.set("collection.configName", "conf1");
}
request = new QueryRequest(params);
request.setPath("/admin/collections");
createNewSolrServer("", baseUrl).request(request);
@ -624,7 +714,8 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
try {
exp = false;
try {
createCollection(collectionInfos, "awholynewcollection_" + cnt, numShards, replicationFactor, maxShardsPerNode, client, null);
createCollection(collectionInfos, "awholynewcollection_" + cnt,
numShards, replicationFactor, maxShardsPerNode, client, null, "conf1");
} catch (SolrException e) {
exp = true;
}
@ -654,7 +745,7 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
collectionInfos = new HashMap<String,List<Integer>>();
client = createCloudClient("awholynewcollection_" + (cnt+1));
try {
createCollection(collectionInfos, "awholynewcollection_" + (cnt+1), numShards, replicationFactor, maxShardsPerNode, client, StrUtils.join(createNodeList, ','));
createCollection(collectionInfos, "awholynewcollection_" + (cnt+1), numShards, replicationFactor, maxShardsPerNode, client, StrUtils.join(createNodeList, ','), "conf1");
} finally {
client.shutdown();
}

10
solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java
View File

@ -59,6 +59,7 @@ public class CoreAdminRequest extends SolrRequest
private String coreNodeName;
private Boolean loadOnStartup;
private Boolean isTransient;
private String collectionConfigName;
public Create() {
action = CoreAdminAction.CREATE;
@ -76,6 +77,7 @@ public class CoreAdminRequest extends SolrRequest
public void setCoreNodeName(String coreNodeName) {this.coreNodeName = coreNodeName;}
public void setIsTransient(Boolean isTransient) { this.isTransient = isTransient; }
public void setIsLoadOnStartup(Boolean loadOnStartup) { this.loadOnStartup = loadOnStartup;}
public void setCollectionConfigName(String name) { this.collectionConfigName = name;}
public String getInstanceDir() { return instanceDir; }
public String getSchemaName() { return schemaName; }
@ -88,7 +90,8 @@ public class CoreAdminRequest extends SolrRequest
public String getCoreNodeName() { return coreNodeName; }
public Boolean getIsLoadOnStartup() { return loadOnStartup; }
public Boolean getIsTransient() { return isTransient; }
public String getCollectionConfigName() { return collectionConfigName;}
@Override
public SolrParams getParams() {
if( action == null ) {
@ -137,6 +140,11 @@ public class CoreAdminRequest extends SolrRequest
if (loadOnStartup != null) {
params.set(CoreAdminParams.LOAD_ON_STARTUP, loadOnStartup);
}
if (collectionConfigName != null) {
params.set("collection.configName", collectionConfigName);
}
return params;
}

23
solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
View File

@ -1537,7 +1537,10 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
createCollection(null, collectionName, numShards, numReplicas, maxShardsPerNode, null, null);
}
protected void createCollection(Map<String,List<Integer>> collectionInfos, String collectionName, Map<String,Object> collectionProps, SolrServer client ) throws SolrServerException, IOException{
protected void createCollection(Map<String,List<Integer>> collectionInfos, String collectionName, Map<String,Object> collectionProps, SolrServer client) throws SolrServerException, IOException{
createCollection(collectionInfos, collectionName, collectionProps, client, null);
}
protected void createCollection(Map<String,List<Integer>> collectionInfos, String collectionName, Map<String,Object> collectionProps, SolrServer client, String confSetName) throws SolrServerException, IOException{
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionAction.CREATE.toString());
for (Map.Entry<String, Object> entry : collectionProps.entrySet()) {
@ -1552,7 +1555,11 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
if(numShards==null){
numShards = (Integer) OverseerCollectionProcessor.COLL_PROPS.get(REPLICATION_FACTOR);
}
if (confSetName != null) {
params.set("collection.configName", confSetName);
}
int clientIndex = random().nextInt(2);
List<Integer> list = new ArrayList<Integer>();
list.add(numShards);
@ -1591,6 +1598,18 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
MAX_SHARDS_PER_NODE, maxShardsPerNode),
client);
}
protected void createCollection(Map<String,List<Integer>> collectionInfos,
String collectionName, int numShards, int numReplicas, int maxShardsPerNode, SolrServer client, String createNodeSetStr, String configName) throws SolrServerException, IOException {
createCollection(collectionInfos, collectionName,
ZkNodeProps.makeMap(
NUM_SLICES, numShards,
REPLICATION_FACTOR, numReplicas,
CREATE_NODE_SET, createNodeSetStr,
MAX_SHARDS_PER_NODE, maxShardsPerNode),
client, configName);
}
@Override
protected SolrServer createNewSolrServer(int port) {

29
solr/test-framework/src/java/org/apache/solr/cloud/AbstractZkTestCase.java
View File

@ -102,27 +102,26 @@ public abstract class AbstractZkTestCase extends SolrTestCaseJ4 {
zkClient.makePath("/collections/control_collection/shards", CreateMode.PERSISTENT, true);
// for now, always upload the config and schema to the canonical names
putConfig(zkClient, solrhome, config, "solrconfig.xml");
putConfig(zkClient, solrhome, schema, "schema.xml");
putConfig("conf1", zkClient, solrhome, config, "solrconfig.xml");
putConfig("conf1", zkClient, solrhome, schema, "schema.xml");
putConfig(zkClient, solrhome, "solrconfig.snippet.randomindexconfig.xml");
putConfig(zkClient, solrhome, "stopwords.txt");
putConfig(zkClient, solrhome, "protwords.txt");
putConfig(zkClient, solrhome, "currency.xml");
putConfig(zkClient, solrhome, "open-exchange-rates.json");
putConfig(zkClient, solrhome, "mapping-ISOLatin1Accent.txt");
putConfig(zkClient, solrhome, "old_synonyms.txt");
putConfig(zkClient, solrhome, "synonyms.txt");
putConfig("conf1", zkClient, solrhome, "solrconfig.snippet.randomindexconfig.xml");
putConfig("conf1", zkClient, solrhome, "stopwords.txt");
putConfig("conf1", zkClient, solrhome, "protwords.txt");
putConfig("conf1", zkClient, solrhome, "currency.xml");
putConfig("conf1", zkClient, solrhome, "open-exchange-rates.json");
putConfig("conf1", zkClient, solrhome, "mapping-ISOLatin1Accent.txt");
putConfig("conf1", zkClient, solrhome, "old_synonyms.txt");
putConfig("conf1", zkClient, solrhome, "synonyms.txt");
zkClient.close();
}
private static void putConfig(SolrZkClient zkClient, File solrhome, final String name)
public static void putConfig(String confName, SolrZkClient zkClient, File solrhome, final String name)
throws Exception {
putConfig(zkClient, solrhome, name, name);
putConfig(confName, zkClient, solrhome, name, name);
}
private static void putConfig(SolrZkClient zkClient, File solrhome, final String srcName, String destName)
public static void putConfig(String confName, SolrZkClient zkClient, File solrhome, final String srcName, String destName)
throws Exception {
File file = new File(solrhome, "collection1"
+ File.separator + "conf" + File.separator + srcName);
@ -131,7 +130,7 @@ public abstract class AbstractZkTestCase extends SolrTestCaseJ4 {
return;
}
String destPath = "/configs/conf1/" + destName;
String destPath = "/configs/" + confName + "/" + destName;
log.info("put " + file.getAbsolutePath() + " to " + destPath);
zkClient.makePath(destPath, file, false, true);
}