Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-10773: Add support for replica types in V2 API

This commit is contained in:
Tomas Fernandez Lobbe 2017-05-31 13:53:34 -07:00
parent 3291ef884d
commit 22c2ed070a
6 changed files with 197 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java
View File

@ -22,6 +22,7 @@ import java.lang.invoke.MethodHandles;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
@ -72,7 +73,7 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
String node = message.getStr(CoreAdminParams.NODE);
String shard = message.getStr(SHARD_ID_PROP);
String coreName = message.getStr(CoreAdminParams.NAME);
Replica.Type replicaType = Replica.Type.valueOf(message.getStr(ZkStateReader.REPLICA_TYPE, Replica.Type.NRT.name()));
Replica.Type replicaType = Replica.Type.valueOf(message.getStr(ZkStateReader.REPLICA_TYPE, Replica.Type.NRT.name()).toUpperCase(Locale.ROOT));
boolean parallel = message.getBool("parallel", false);
if (StringUtils.isBlank(coreName)) {
coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME);

14
solr/core/src/resources/apispec/collections.Commands.json
View File

@ -51,7 +51,19 @@
},
"replicationFactor": {
"type": "integer",
"description": "The number of replicas to be created for each shard. Replicas are physical copies of each shard, acting as failover for the shard."
"description": "The number of NRT replicas to be created for each shard. Replicas are physical copies of each shard, acting as failover for the shard."
},
"nrtReplicas": {
"type": "integer",
"description": "The number of NRT replicas to be created for each shard. Replicas are physical copies of each shard, acting as failover for the shard. Replicas of type NRT will be updated with each document that is added to the cluster, and can use \"softCommits\" to get a new view of the index in Near Real Time. This parameter works in the same way as 'replicationFactor'"
},
"tlogReplicas": {
"type": "integer",
"description": "The number of TLOG replicas to be created for each shard. TLOG replicas update their transaction log for every update to the cluster, but only the shard leader updates the local index, other TLOG replicas will use segment replication and copy the latest index files from the leader."
},
"pullReplicas": {
"type": "integer",
"description": "The number of PULL replicas to be created for each shard. PULL replicas don't receive copies of the documents on update requests, they just replicate the latest segments periodically from the shard leader. PULL replicas can't become shard leaders, and need at least one active TLOG(recommended) or NRT replicas in the shard to replicate from."
},
"nodeSet": {
"type": "array",

5
solr/core/src/resources/apispec/collections.collection.shards.Commands.json
View File

@ -101,6 +101,11 @@
"async": {
"type": "string",
"description": "Defines a request ID that can be used to track this action after it's submitted. The action will be processed asynchronously when this is defined."
},
"type": {
"type": "string",
"enum":["NRT", "TLOG", "PULL"],
"description": "The type of replica to add. NRT (default), TLOG or PULL"
}
},
"required":["shard"]

89
solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java
View File

@ -28,8 +28,12 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
@ -117,21 +121,40 @@ public class TestPullReplica extends SolrCloudTestCase {
@Repeat(iterations=2) // 2 times to make sure cleanup is complete and we can create the same collection
public void testCreateDelete() throws Exception {
try {
if (random().nextBoolean()) {
CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1, 0, 3)
.setMaxShardsPerNode(100)
.process(cluster.getSolrClient());
} else {
// Sometimes don't use SolrJ.
String url = String.format(Locale.ROOT, "%s/admin/collections?action=CREATE&name=%s&numShards=%s&pullReplicas=%s&maxShardsPerNode=%s",
cluster.getRandomJetty(random()).getBaseUrl(),
collectionName,
2, // numShards
3, // pullReplicas
100); // maxShardsPerNode
url = url + pickRandom("", "&nrtReplicas=1", "&replicationFactor=1"); // These options should all mean the same
HttpGet createCollectionRequest = new HttpGet(url);
cluster.getSolrClient().getHttpClient().execute(createCollectionRequest);
switch (random().nextInt(3)) {
case 0:
// Sometimes use SolrJ
CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1, 0, 3)
.setMaxShardsPerNode(100)
.process(cluster.getSolrClient());
break;
case 1:
// Sometimes use v1 API
String url = String.format(Locale.ROOT, "%s/admin/collections?action=CREATE&name=%s&numShards=%s&pullReplicas=%s&maxShardsPerNode=%s",
cluster.getRandomJetty(random()).getBaseUrl(),
collectionName,
2, // numShards
3, // pullReplicas
100); // maxShardsPerNode
url = url + pickRandom("", "&nrtReplicas=1", "&replicationFactor=1"); // These options should all mean the same
HttpGet createCollectionGet = new HttpGet(url);
cluster.getSolrClient().getHttpClient().execute(createCollectionGet);
break;
case 2:
// Sometimes use V2 API
url = cluster.getRandomJetty(random()).getBaseUrl().toString() + "/____v2/c";
String requestBody = String.format(Locale.ROOT, "{create:{name:%s, numShards:%s, pullReplicas:%s, maxShardsPerNode:%s %s}}",
collectionName,
2, // numShards
3, // pullReplicas
100, // maxShardsPerNode
pickRandom("", ", nrtReplicas:1", ", replicationFactor:1")); // These options should all mean the same
HttpPost createCollectionPost = new HttpPost(url);
createCollectionPost.setHeader("Content-type", "application/json");
createCollectionPost.setEntity(new StringEntity(requestBody));
HttpResponse httpResponse = cluster.getSolrClient().getHttpClient().execute(createCollectionPost);
assertEquals(200, httpResponse.getStatusLine().getStatusCode());
break;
}
boolean reloaded = false;
while (true) {
@ -243,9 +266,9 @@ public class TestPullReplica extends SolrCloudTestCase {
DocCollection docCollection = assertNumberOfReplicas(2, 0, 0, false, true);
assertEquals(2, docCollection.getSlices().size());
CollectionAdminRequest.addReplicaToShard(collectionName, "shard1", Replica.Type.PULL).process(cluster.getSolrClient());
addReplicaToShard("shard1", Replica.Type.PULL);
docCollection = assertNumberOfReplicas(2, 0, 1, true, false);
CollectionAdminRequest.addReplicaToShard(collectionName, "shard2", Replica.Type.PULL).process(cluster.getSolrClient());
addReplicaToShard("shard2", Replica.Type.PULL);
docCollection = assertNumberOfReplicas(2, 0, 2, true, false);
waitForState("Expecting collection to have 2 shards and 2 replica each", collectionName, clusterShape(2, 2));
@ -587,4 +610,36 @@ public class TestPullReplica extends SolrCloudTestCase {
cluster.getSolrClient().add(collectionName, docs);
cluster.getSolrClient().commit(collectionName);
}
private void addReplicaToShard(String shardName, Replica.Type type) throws ClientProtocolException, IOException, SolrServerException {
switch (random().nextInt(3)) {
case 0: // Add replica with SolrJ
CollectionAdminResponse response = CollectionAdminRequest.addReplicaToShard(collectionName, shardName, type).process(cluster.getSolrClient());
assertEquals("Unexpected response status: " + response.getStatus(), 0, response.getStatus());
break;
case 1: // Add replica with V1 API
String url = String.format(Locale.ROOT, "%s/admin/collections?action=ADDREPLICA&collection=%s&shard=%s&type=%s",
cluster.getRandomJetty(random()).getBaseUrl(),
collectionName,
shardName,
type);
HttpGet addReplicaGet = new HttpGet(url);
HttpResponse httpResponse = cluster.getSolrClient().getHttpClient().execute(addReplicaGet);
assertEquals(200, httpResponse.getStatusLine().getStatusCode());
break;
case 2:// Add replica with V2 API
url = String.format(Locale.ROOT, "%s/____v2/c/%s/shards",
cluster.getRandomJetty(random()).getBaseUrl(),
collectionName);
String requestBody = String.format(Locale.ROOT, "{add-replica:{shard:%s, type:%s}}",
shardName,
type);
HttpPost addReplicaPost = new HttpPost(url);
addReplicaPost.setHeader("Content-type", "application/json");
addReplicaPost.setEntity(new StringEntity(requestBody));
httpResponse = cluster.getSolrClient().getHttpClient().execute(addReplicaPost);
assertEquals(200, httpResponse.getStatusLine().getStatusCode());
break;
}
}
}

86
solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java
View File

@ -29,8 +29,12 @@ import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrClient;
@ -145,20 +149,38 @@ public class TestTlogReplica extends SolrCloudTestCase {
@Repeat(iterations=2) // 2 times to make sure cleanup is complete and we can create the same collection
public void testCreateDelete() throws Exception {
try {
if (random().nextBoolean()) {
CollectionAdminRequest.createCollection(collectionName, "conf", 2, 0, 4, 0)
.setMaxShardsPerNode(100)
.process(cluster.getSolrClient());
} else {
// Sometimes don't use SolrJ
String url = String.format(Locale.ROOT, "%s/admin/collections?action=CREATE&name=%s&numShards=%s&tlogReplicas=%s&maxShardsPerNode=%s",
cluster.getRandomJetty(random()).getBaseUrl(),
collectionName,
2, // numShards
4, // tlogReplicas
100); // maxShardsPerNode
HttpGet createCollectionRequest = new HttpGet(url);
cluster.getSolrClient().getHttpClient().execute(createCollectionRequest);
switch (random().nextInt(3)) {
case 0:
CollectionAdminRequest.createCollection(collectionName, "conf", 2, 0, 4, 0)
.setMaxShardsPerNode(100)
.process(cluster.getSolrClient());
break;
case 1:
// Sometimes don't use SolrJ
String url = String.format(Locale.ROOT, "%s/admin/collections?action=CREATE&name=%s&numShards=%s&tlogReplicas=%s&maxShardsPerNode=%s",
cluster.getRandomJetty(random()).getBaseUrl(),
collectionName,
2, // numShards
4, // tlogReplicas
100); // maxShardsPerNode
HttpGet createCollectionGet = new HttpGet(url);
HttpResponse httpResponse = cluster.getSolrClient().getHttpClient().execute(createCollectionGet);
assertEquals(200, httpResponse.getStatusLine().getStatusCode());
break;
case 2:
// Sometimes use V2 API
url = cluster.getRandomJetty(random()).getBaseUrl().toString() + "/____v2/c";
String requestBody = String.format(Locale.ROOT, "{create:{name:%s, numShards:%s, tlogReplicas:%s, maxShardsPerNode:%s}}",
collectionName,
2, // numShards
4, // tlogReplicas
100); // maxShardsPerNode
HttpPost createCollectionPost = new HttpPost(url);
createCollectionPost.setHeader("Content-type", "application/json");
createCollectionPost.setEntity(new StringEntity(requestBody));
httpResponse = cluster.getSolrClient().getHttpClient().execute(createCollectionPost);
assertEquals(200, httpResponse.getStatusLine().getStatusCode());
break;
}
boolean reloaded = false;
@ -244,9 +266,9 @@ public class TestTlogReplica extends SolrCloudTestCase {
DocCollection docCollection = createAndWaitForCollection(2, 0, 1, 0);
assertEquals(2, docCollection.getSlices().size());
CollectionAdminRequest.addReplicaToShard(collectionName, "shard1", Replica.Type.TLOG).process(cluster.getSolrClient());
addReplicaToShard("shard1", Replica.Type.TLOG);
docCollection = assertNumberOfReplicas(0, 3, 0, true, false);
CollectionAdminRequest.addReplicaToShard(collectionName, "shard2", Replica.Type.TLOG).process(cluster.getSolrClient());
addReplicaToShard("shard2", Replica.Type.TLOG);
docCollection = assertNumberOfReplicas(0, 4, 0, true, false);
waitForState("Expecting collection to have 2 shards and 2 replica each", collectionName, clusterShape(2, 2));
@ -260,6 +282,38 @@ public class TestTlogReplica extends SolrCloudTestCase {
assertNumberOfReplicas(0, 3, 0, true, true);
}
private void addReplicaToShard(String shardName, Replica.Type type) throws ClientProtocolException, IOException, SolrServerException {
switch (random().nextInt(3)) {
case 0: // Add replica with SolrJ
CollectionAdminResponse response = CollectionAdminRequest.addReplicaToShard(collectionName, shardName, type).process(cluster.getSolrClient());
assertEquals("Unexpected response status: " + response.getStatus(), 0, response.getStatus());
break;
case 1: // Add replica with V1 API
String url = String.format(Locale.ROOT, "%s/admin/collections?action=ADDREPLICA&collection=%s&shard=%s&type=%s",
cluster.getRandomJetty(random()).getBaseUrl(),
collectionName,
shardName,
type);
HttpGet addReplicaGet = new HttpGet(url);
HttpResponse httpResponse = cluster.getSolrClient().getHttpClient().execute(addReplicaGet);
assertEquals(200, httpResponse.getStatusLine().getStatusCode());
break;
case 2:// Add replica with V2 API
url = String.format(Locale.ROOT, "%s/____v2/c/%s/shards",
cluster.getRandomJetty(random()).getBaseUrl(),
collectionName);
String requestBody = String.format(Locale.ROOT, "{add-replica:{shard:%s, type:%s}}",
shardName,
type);
HttpPost addReplicaPost = new HttpPost(url);
addReplicaPost.setHeader("Content-type", "application/json");
addReplicaPost.setEntity(new StringEntity(requestBody));
httpResponse = cluster.getSolrClient().getHttpClient().execute(addReplicaPost);
assertEquals(200, httpResponse.getStatusLine().getStatusCode());
break;
}
}
public void testRemoveLeader() throws Exception {
doReplaceLeader(true);
}

38
solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java
View File

@ -58,11 +58,19 @@ public class TestCollectionAPIs extends SolrTestCaseJ4 {
MockCollectionsHandler collectionsHandler = new MockCollectionsHandler();
ApiBag apiBag = new ApiBag(false);
Collection<Api> apis = collectionsHandler.getApis();
for (Api api : apis) apiBag.register(api, Collections.EMPTY_MAP);
for (Api api : apis) apiBag.register(api, Collections.emptyMap());
//test a simple create collection call
compareOutput(apiBag, "/collections", POST,
"{create:{name:'newcoll', config:'schemaless', numShards:2, replicationFactor:2 }}", null,
"{name:newcoll, fromApi:'true', replicationFactor:'2', collection.configName:schemaless, numShards:'2', stateFormat:'2', operation:create}");
compareOutput(apiBag, "/collections", POST,
"{create:{name:'newcoll', config:'schemaless', numShards:2, nrtReplicas:2 }}", null,
"{name:newcoll, fromApi:'true', nrtReplicas:'2', collection.configName:schemaless, numShards:'2', stateFormat:'2', operation:create}");
compareOutput(apiBag, "/collections", POST,
"{create:{name:'newcoll', config:'schemaless', numShards:2, nrtReplicas:2, tlogReplicas:2, pullReplicas:2 }}", null,
"{name:newcoll, fromApi:'true', nrtReplicas:'2', tlogReplicas:'2', pullReplicas:'2', collection.configName:schemaless, numShards:'2', stateFormat:'2', operation:create}");
//test a create collection with custom properties
compareOutput(apiBag, "/collections", POST,
@ -106,6 +114,21 @@ public class TestCollectionAPIs extends SolrTestCaseJ4 {
"{split:{ splitKey:id12345, coreProperties : {prop1:prop1Val, prop2:prop2Val} }}", null,
"{collection: collName , split.key : id12345 , operation : splitshard, property.prop1:prop1Val, property.prop2: prop2Val}"
);
compareOutput(apiBag, "/collections/collName/shards", POST,
"{add-replica:{shard: shard1, node: 'localhost_8978' , type:'TLOG' }}", null,
"{collection: collName , shard : shard1, node :'localhost_8978', operation : addreplica, type: TLOG}"
);
compareOutput(apiBag, "/collections/collName/shards", POST,
"{add-replica:{shard: shard1, node: 'localhost_8978' , type:'PULL' }}", null,
"{collection: collName , shard : shard1, node :'localhost_8978', operation : addreplica, type: PULL}"
);
assertErrorContains(apiBag, "/collections/collName/shards", POST,
"{add-replica:{shard: shard1, node: 'localhost_8978' , type:'foo' }}", null,
"Value of enum must be one of"
);
compareOutput(apiBag, "/collections/collName", POST,
"{add-replica-property : {name:propA , value: VALA, shard: shard1, replica:replica1}}", null,
@ -150,13 +173,22 @@ public class TestCollectionAPIs extends SolrTestCaseJ4 {
Map expected = (Map) fromJSONString(expectedOutputMapJson);
assertMapEqual(expected, output);
return output;
}
static void assertErrorContains(final ApiBag apiBag, final String path, final SolrRequest.METHOD method,
final String payload, final CoreContainer cc, String expectedErrorMsg) throws Exception {
try {
makeCall(apiBag, path, method, payload, cc);
fail("Expected exception");
} catch (RuntimeException e) {
assertTrue("Expected exception with error message '" + expectedErrorMsg + "' but got: " + e.getMessage(), e.getMessage().contains(expectedErrorMsg));
}
}
public static Pair<SolrQueryRequest, SolrQueryResponse> makeCall(final ApiBag apiBag, String path,
final SolrRequest.METHOD method,
final String payload, final CoreContainer cc) throws Exception {
SolrParams queryParams = new MultiMapSolrParams(Collections.EMPTY_MAP);
SolrParams queryParams = new MultiMapSolrParams(Collections.emptyMap());
if (path.indexOf('?') > 0) {
String queryStr = path.substring(path.indexOf('?') + 1);
path = path.substring(0, path.indexOf('?'));