Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-3108: Error in SolrCloud's replica lookup code when replica's are hosted in same Solr instance (test) 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1242324 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2012-02-09 13:59:11 +00:00
parent 25cfcfb61e
commit 3c5d8633d9
10 changed files with 124 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -108,6 +108,8 @@ New Features
(Soheb Mahmood, Alex Cowell, Mark Miller)
SOLR-2622: ShowFileRequestHandler does not work in SolrCloud mode.
(Stefan Matheis, Mark Miller)
SOLR-3108: Error in SolrCloud's replica lookup code when replica's are hosted in same Solr instance.
(Bruno Dumon, Sami Siren, Mark Miller)
* SOLR-1566: Transforming documents in the ResponseWriters. This will allow
for more complex results in responses and open the door for function queries

12
solr/core/src/java/org/apache/solr/cloud/AssignShard.java
View File

@ -26,8 +26,6 @@ import java.util.Map;
import org.apache.solr.common.cloud.CloudState;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.zookeeper.KeeperException;
public class AssignShard {
@ -38,10 +36,10 @@ public class AssignShard {
* @param state
* @return the assigned shard id
*/
public static String assignShard(String collection, CloudState state) {
int shards = Integer.getInteger(ZkStateReader.NUM_SHARDS_PROP,1);
public static String assignShard(String collection, CloudState state, Integer numShards) {
if (numShards == null) {
numShards = 1;
}
String returnShardId = null;
Map<String, Slice> sliceMap = state.getSlices(collection);
@ -51,7 +49,7 @@ public class AssignShard {
List<String> shardIdNames = new ArrayList<String>(sliceMap.keySet());
if (shardIdNames.size() < shards) {
if (shardIdNames.size() < numShards) {
return "shard" + (shardIdNames.size() + 1);
}

11
solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
View File

@ -24,6 +24,7 @@ public class CloudDescriptor {
private String collectionName;
private SolrParams params;
private String roles = "";
private Integer numShards;
public void setShardId(String shardId) {
this.shardId = shardId;
@ -57,4 +58,14 @@ public class CloudDescriptor {
public void setParams(SolrParams params) {
this.params = params;
}
// setting only matters on core creation
public Integer getNumShards() {
return numShards;
}
public void setNumShards(int numShards) {
this.numShards = numShards;
}
}

2
solr/core/src/java/org/apache/solr/cloud/Overseer.java
View File

@ -198,7 +198,7 @@ public class Overseer implements NodeStateChangeListener, ShardLeaderListener {
}
if(shardId==null) {
//request new shardId
shardId = AssignShard.assignShard(collection, state);
shardId = AssignShard.assignShard(collection, state, coreState.getNumShards());
}
Map<String,String> props = new HashMap<String,String>();

7
solr/core/src/java/org/apache/solr/cloud/ZkController.java
View File

@ -844,9 +844,12 @@ public final class ZkController {
private void doPublish(String shardZkNodeName, String coreName,
Map<String,String> props, CloudDescriptor cloudDesc) {
Integer numShards = cloudDesc.getNumShards();
if (numShards == null) {
numShards = Integer.getInteger(ZkStateReader.NUM_SHARDS_PROP);
}
CoreState coreState = new CoreState(coreName,
cloudDesc.getCollectionName(), props);
cloudDesc.getCollectionName(), props, numShards);
coreStates.put(shardZkNodeName, coreState);
final String nodePath = "/node_states/" + getNodeName();

5
solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
View File

@ -26,7 +26,6 @@ import java.util.Properties;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.client.solrj.SolrServerException;
@ -354,6 +353,10 @@ public class CoreAdminHandler extends RequestHandlerBase {
opts = params.get(CoreAdminParams.SHARD);
if (opts != null)
cd.setShardId(opts);
Integer numShards = params.getInt(ZkStateReader.NUM_SHARDS_PROP);
if (numShards != null)
cd.setNumShards(numShards);
}
// Process all property.name=value parameters and set them as name=value core properties

61
solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
View File

@ -61,6 +61,7 @@ public class BasicDistributedZkTest extends AbstractDistributedZkTestCase {
String invalidField="ignore_exception__invalid_field_not_in_schema";
private Map<String,List<SolrServer>> otherCollectionClients = new HashMap<String,List<SolrServer>>();
private Map<String,List<SolrServer>> oneInstanceCollectionClients = new HashMap<String,List<SolrServer>>();
public BasicDistributedZkTest() {
fixShardCount = true;
@ -243,12 +244,72 @@ public class BasicDistributedZkTest extends AbstractDistributedZkTestCase {
// query("q","matchesnothing","fl","*,score", "debugQuery", "true");
testMultipleCollections();
testANewCollectionInOneInstance();
// Thread.sleep(10000000000L);
if (DEBUG) {
super.printLayout();
}
}
private void testANewCollectionInOneInstance() throws Exception {
String collection = "oneInstanceCollection";
List<SolrServer> collectionClients = new ArrayList<SolrServer>();
SolrServer client = clients.get(0);
oneInstanceCollectionClients.put(collection, collectionClients);
String baseUrl = ((CommonsHttpSolrServer) client).getBaseURL();
createCollection(collection, collectionClients, baseUrl, 1);
createCollection(collection, collectionClients, baseUrl, 2);
createCollection(collection, collectionClients, baseUrl, 3);
createCollection(collection, collectionClients, baseUrl, 4);
SolrServer client1 = createNewSolrServer(collection + "1", baseUrl);
SolrServer client2 = createNewSolrServer(collection + "2", baseUrl);
SolrServer client3 = createNewSolrServer(collection + "3", baseUrl);
SolrServer client4 = createNewSolrServer(collection + "4", baseUrl);
client2.add(getDoc(id, "1"));
client3.add(getDoc(id, "2"));
client4.add(getDoc(id, "3"));
client1.commit();
SolrQuery query = new SolrQuery("*:*");
query.set("distrib", false);
long oneDocs = client1.query(query).getResults().getNumFound();
long twoDocs = client2.query(query).getResults().getNumFound();
long threeDocs = client3.query(query).getResults().getNumFound();
long fourDocs = client4.query(query).getResults().getNumFound();
query.set("collection", collection);
query.set("distrib", true);
long allDocs = solrj.query(query).getResults().getNumFound();
// System.out.println("1:" + oneDocs);
// System.out.println("2:" + twoDocs);
// System.out.println("3:" + threeDocs);
// System.out.println("4:" + fourDocs);
// System.out.println("All Docs:" + allDocs);
assertEquals(oneDocs, threeDocs);
assertEquals(twoDocs, fourDocs);
assertNotSame(oneDocs, twoDocs);
assertEquals(3, allDocs);
}
private void createCollection(String collection,
List<SolrServer> collectionClients, String baseUrl, int num)
throws MalformedURLException, SolrServerException, IOException {
CommonsHttpSolrServer server = new CommonsHttpSolrServer(
baseUrl);
Create createCmd = new Create();
createCmd.setCoreName(collection + num);
createCmd.setCollection(collection);
createCmd.setNumShards(2);
createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
+ collection + num);
server.request(createCmd);
collectionClients.add(createNewSolrServer(collection, baseUrl));
}
private void testMultipleCollections() throws MalformedURLException,
SolrServerException, IOException, Exception {
// create another 2 collections and search across them

20
solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
View File

@ -86,12 +86,12 @@ public class OverseerTest extends SolrTestCaseJ4 {
}
}
public void publishState(String coreName, String stateName) throws KeeperException, InterruptedException{
public void publishState(String coreName, String stateName, int numShards) throws KeeperException, InterruptedException{
HashMap<String,String> coreProps = new HashMap<String,String>();
coreProps.put(ZkStateReader.STATE_PROP, stateName);
coreProps.put(ZkStateReader.NODE_NAME_PROP, nodeName);
coreProps.put(ZkStateReader.CORE_NAME_PROP, coreName);
CoreState state = new CoreState(coreName, "collection1", coreProps);
CoreState state = new CoreState(coreName, "collection1", coreProps, numShards);
final String statePath = Overseer.STATES_NODE + "/" + nodeName;
zkClient.setData(statePath, ZkStateReader.toJSON(new CoreState[] {state}), true);
}
@ -397,8 +397,6 @@ public class OverseerTest extends SolrTestCaseJ4 {
AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
zkClient.makePath("/live_nodes", true);
System.setProperty(ZkStateReader.NUM_SHARDS_PROP, "2");
//live node
String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + "node1";
zkClient.makePath(nodePath,CreateMode.EPHEMERAL, true);
@ -416,7 +414,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
coreProps.put(ZkStateReader.CORE_NAME_PROP, "core1");
coreProps.put(ZkStateReader.ROLES_PROP, "");
coreProps.put(ZkStateReader.STATE_PROP, ZkStateReader.RECOVERING);
CoreState state = new CoreState("core1", "collection1", coreProps);
CoreState state = new CoreState("core1", "collection1", coreProps, 2);
nodePath = "/node_states/node1";
@ -441,7 +439,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
coreProps.put(ZkStateReader.STATE_PROP, ZkStateReader.ACTIVE);
coreProps.put(ZkStateReader.SHARD_ID_PROP, "shard1");
state = new CoreState("core1", "collection1", coreProps);
state = new CoreState("core1", "collection1", coreProps, 2);
zkClient.setData(nodePath, ZkStateReader.toJSON(new CoreState[]{state}), true);
@ -503,7 +501,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
overseerClient = electNewOverseer(server.getZkAddress());
mockController.publishState("core1", ZkStateReader.RECOVERING);
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
// wait overseer assignment
waitForSliceCount(reader, "collection1", 1);
@ -512,7 +510,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
int version = getCloudStateVersion(controllerClient);
mockController.publishState("core1", ZkStateReader.ACTIVE);
mockController.publishState("core1", ZkStateReader.ACTIVE, 1);
while(version == getCloudStateVersion(controllerClient));
@ -521,7 +519,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
overseerClient.close();
Thread.sleep(1000); //wait for overseer to get killed
mockController.publishState("core1", ZkStateReader.RECOVERING);
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
version = getCloudStateVersion(controllerClient);
overseerClient = electNewOverseer(server.getZkAddress());
@ -582,7 +580,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
overseerClient = electNewOverseer(server.getZkAddress());
mockController.publishState("core1", ZkStateReader.RECOVERING);
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
// wait overseer assignment
waitForSliceCount(reader, "collection1", 1);
@ -594,7 +592,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
int version = getCloudStateVersion(controllerClient);
mockController = new MockZKController(server.getZkAddress(), "node1");
mockController.publishState("core1", ZkStateReader.RECOVERING);
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
while (version == getCloudStateVersion(controllerClient));

15
solr/solrj/src/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java
View File

@ -26,6 +26,7 @@ import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.SolrParams;
@ -49,6 +50,8 @@ public class CoreAdminRequest extends SolrRequest
protected String configName = null;
protected String schemaName = null;
protected String dataDir = null;
protected String collection;
private Integer numShards;
public Create() {
action = CoreAdminAction.CREATE;
@ -58,12 +61,15 @@ public class CoreAdminRequest extends SolrRequest
public void setSchemaName(String schema) { this.schemaName = schema; }
public void setConfigName(String config) { this.configName = config; }
public void setDataDir(String dataDir) { this.dataDir = dataDir; }
public void setCollection(String collection) { this.collection = collection; }
public void setNumShards(int numShards) {this.numShards = numShards;}
public String getInstanceDir() { return instanceDir; }
public String getSchemaName() { return schemaName; }
public String getConfigName() { return configName; }
public String getDataDir() { return dataDir; }
public String getCollection() { return collection; }
@Override
public SolrParams getParams() {
if( action == null ) {
@ -86,8 +92,15 @@ public class CoreAdminRequest extends SolrRequest
if (dataDir != null) {
params.set( CoreAdminParams.DATA_DIR, dataDir);
}
if (collection != null) {
params.set( CoreAdminParams.COLLECTION, collection);
}
if (numShards != null) {
params.set( ZkStateReader.NUM_SHARDS_PROP, numShards);
}
return params;
}
}
public static class PrepRecovery extends CoreAdminRequest {

13
solr/solrj/src/java/org/apache/solr/common/cloud/CoreState.java
View File

@ -32,11 +32,14 @@ public class CoreState implements JSONWriter.Writable {
this.properties = Collections.unmodifiableMap(props);
}
public CoreState(String coreName, String collectionName, Map<String,String> properties) {
public CoreState(String coreName, String collectionName, Map<String,String> properties, Integer numShards) {
HashMap<String,String> props = new HashMap<String,String>();
props.putAll(properties);
props.put(ZkStateReader.COLLECTION_PROP, collectionName);
props.put(ZkStateReader.CORE_NAME_PROP, coreName);
if (numShards != null) {
props.put("num_shards", Integer.toString(numShards));
}
this.properties = Collections.unmodifiableMap(props);
}
@ -93,4 +96,12 @@ public class CoreState implements JSONWriter.Writable {
return "coll:" + getCollectionName() + " core:" + getCoreName() + " props:" + properties;
}
public Integer getNumShards() {
String numShards = properties.get("num_shards");
if (numShards == null) {
return null;
}
return Integer.parseInt(numShards);
}
}