Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-11045: The new replica created by MoveReplica will have to have same name and coreName as the old one in case of HDFS

This commit is contained in:
Cao Manh Dat 2017-07-13 13:43:21 +07:00
parent 8b65515f07
commit 22ec456b0c
5 changed files with 72 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -334,6 +334,9 @@ Bug Fixes
* SOLR-11041: MoveReplicaCmd do not specify ulog dir in case of HDFS (Cao Manh Dat)
* SOLR-11045: The new replica created by MoveReplica will have to have same name and coreName as the
old one in case of HDFS (Cao Manh Dat)
Optimizations
----------------------

4
solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
View File

@ -31,6 +31,7 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrResourceLoader;
@ -64,10 +65,11 @@ public class CloudUtil {
String cnn = replica.getName();
String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
boolean isSharedFs = replica.getStr(CoreAdminParams.DATA_DIR) != null;
log.debug("compare against coreNodeName={} baseUrl={}", cnn, baseUrl);
if (thisCnn != null && thisCnn.equals(cnn)
&& !thisBaseUrl.equals(baseUrl)) {
&& !thisBaseUrl.equals(baseUrl) && isSharedFs) {
if (cc.getLoadedCoreNames().contains(desc.getName())) {
cc.unload(desc.getName());
}

24
solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java
View File

@ -35,6 +35,7 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
import org.apache.solr.update.UpdateLog;
import org.apache.solr.util.TimeOut;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -115,8 +116,6 @@ public class MoveReplicaCmd implements Cmd{
private void moveHdfsReplica(ClusterState clusterState, NamedList results, String dataDir, String targetNode, String async,
DocCollection coll, Replica replica, Slice slice, int timeout) throws Exception {
String newCoreName = Assign.buildCoreName(coll, slice.getName(), replica.getType());
ZkNodeProps removeReplicasProps = new ZkNodeProps(
COLLECTION_PROP, coll.getName(),
SHARD_ID_PROP, slice.getName(),
@ -135,17 +134,32 @@ public class MoveReplicaCmd implements Cmd{
return;
}
TimeOut timeOut = new TimeOut(20L, TimeUnit.SECONDS);
while (!timeOut.hasTimedOut()) {
coll = ocmh.zkStateReader.getClusterState().getCollection(coll.getName());
if (coll.getReplica(replica.getName()) != null) {
Thread.sleep(100);
} else {
break;
}
}
if (timeOut.hasTimedOut()) {
results.add("failure", "Still see deleted replica in clusterstate!");
return;
}
String ulogDir = replica.getStr(CoreAdminParams.ULOG_DIR);
ZkNodeProps addReplicasProps = new ZkNodeProps(
COLLECTION_PROP, coll.getName(),
SHARD_ID_PROP, slice.getName(),
CoreAdminParams.NODE, targetNode,
CoreAdminParams.NAME, newCoreName,
CoreAdminParams.CORE_NODE_NAME, replica.getName(),
CoreAdminParams.NAME, replica.getCoreName(),
CoreAdminParams.ULOG_DIR, ulogDir.substring(0, ulogDir.lastIndexOf(UpdateLog.TLOG_NAME)),
CoreAdminParams.DATA_DIR, dataDir);
if(async!=null) addReplicasProps.getProperties().put(ASYNC, async);
NamedList addResult = new NamedList();
ocmh.addReplica(clusterState, addReplicasProps, addResult, null);
ocmh.addReplica(ocmh.zkStateReader.getClusterState(), addReplicasProps, addResult, null);
if (addResult.get("failure") != null) {
String errorString = String.format(Locale.ROOT, "Failed to create replica for collection=%s shard=%s" +
" on node=%s", coll.getName(), slice.getName(), targetNode);
@ -154,7 +168,7 @@ public class MoveReplicaCmd implements Cmd{
return;
} else {
String successString = String.format(Locale.ROOT, "MOVEREPLICA action completed successfully, moved replica=%s at node=%s " +
"to replica=%s at node=%s", replica.getCoreName(), replica.getNodeName(), newCoreName, targetNode);
"to replica=%s at node=%s", replica.getCoreName(), replica.getNodeName(), replica.getCoreName(), targetNode);
results.add("success", successString);
}
}

5
solr/core/src/java/org/apache/solr/cloud/ZkController.java
View File

@ -2184,10 +2184,7 @@ public class ZkController {
DocCollection collection = clusterState.getCollectionOrNull(desc
.getCloudDescriptor().getCollectionName());
if (collection != null) {
boolean autoAddReplicas = ClusterStateUtil.isAutoAddReplicas(getZkStateReader(), collection.getName());
if (autoAddReplicas) {
CloudUtil.checkSharedFSFailoverReplaced(cc, desc);
}
CloudUtil.checkSharedFSFailoverReplaced(cc, desc);
}
}
}

53
solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSUlogDirTest.java
View File

@ -21,10 +21,13 @@ import java.io.IOException;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterStateUtil;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
@ -65,7 +68,7 @@ public class MoveReplicaHDFSUlogDirTest extends SolrCloudTestCase {
}
@Test
public void testDataDirAndUlogAreMaintained() throws IOException, SolrServerException {
public void testDataDirAndUlogAreMaintained() throws Exception {
String coll = "movereplicatest_coll2";
CollectionAdminRequest.createCollection(coll, "conf1", 1, 1)
.setCreateNodeSet("")
@ -82,7 +85,8 @@ public class MoveReplicaHDFSUlogDirTest extends SolrCloudTestCase {
ulogDir += "/tlog";
ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000);
assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
DocCollection docCollection = zkStateReader.getClusterState().getCollection(coll);
Replica replica = docCollection.getReplicas().iterator().next();
assertTrue(replica.getStr("ulogDir"), replica.getStr("ulogDir").equals(ulogDir) || replica.getStr("ulogDir").equals(ulogDir+'/'));
@ -90,14 +94,49 @@ public class MoveReplicaHDFSUlogDirTest extends SolrCloudTestCase {
new CollectionAdminRequest.MoveReplica(coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName())
.process(cluster.getSolrClient());
ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000);
assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
docCollection = zkStateReader.getClusterState().getCollection(coll);
assertEquals(1, docCollection.getSlice("shard1").getReplicas().size());
replica = docCollection.getReplicas().iterator().next();
assertEquals(replica.getNodeName(), cluster.getJettySolrRunner(1).getNodeName());
assertTrue(replica.getStr("ulogDir"), replica.getStr("ulogDir").equals(ulogDir) || replica.getStr("ulogDir").equals(ulogDir+'/'));
assertTrue(replica.getStr("dataDir"),replica.getStr("dataDir").equals(dataDir) || replica.getStr("dataDir").equals(dataDir+'/'));
Replica newReplica = docCollection.getReplicas().iterator().next();
assertEquals(newReplica.getNodeName(), cluster.getJettySolrRunner(1).getNodeName());
assertTrue(newReplica.getStr("ulogDir"), newReplica.getStr("ulogDir").equals(ulogDir) || newReplica.getStr("ulogDir").equals(ulogDir+'/'));
assertTrue(newReplica.getStr("dataDir"),newReplica.getStr("dataDir").equals(dataDir) || newReplica.getStr("dataDir").equals(dataDir+'/'));
assertEquals(replica.getName(), newReplica.getName());
assertEquals(replica.getCoreName(), newReplica.getCoreName());
assertFalse(replica.getNodeName().equals(newReplica.getNodeName()));
final int numDocs = 100;
addDocs(coll, numDocs); // indexed but not committed
cluster.getJettySolrRunner(1).stop();
Thread.sleep(5000);
new CollectionAdminRequest.MoveReplica(coll, newReplica.getName(), cluster.getJettySolrRunner(0).getNodeName())
.process(cluster.getSolrClient());
assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
// assert that the old core will be removed on startup
cluster.getJettySolrRunner(1).start();
assertTrue(ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000));
docCollection = zkStateReader.getClusterState().getCollection(coll);
assertEquals(1, docCollection.getReplicas().size());
newReplica = docCollection.getReplicas().iterator().next();
assertEquals(newReplica.getNodeName(), cluster.getJettySolrRunner(0).getNodeName());
assertTrue(newReplica.getStr("ulogDir"), newReplica.getStr("ulogDir").equals(ulogDir) || newReplica.getStr("ulogDir").equals(ulogDir+'/'));
assertTrue(newReplica.getStr("dataDir"),newReplica.getStr("dataDir").equals(dataDir) || newReplica.getStr("dataDir").equals(dataDir+'/'));
assertEquals(0, cluster.getJettySolrRunner(1).getCoreContainer().getCores().size());
cluster.getSolrClient().commit(coll);
assertEquals(numDocs, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
}
private void addDocs(String collection, int numDocs) throws SolrServerException, IOException {
SolrClient solrClient = cluster.getSolrClient();
for (int docId = 1; docId <= numDocs; docId++) {
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", docId);
solrClient.add(collection, doc);
}
}
}