From ad7f2a787ddbd978af967a535795e28e1fa70409 Mon Sep 17 00:00:00 2001 From: Cao Manh Dat Date: Wed, 12 Jul 2017 14:07:27 +0700 Subject: [PATCH] SOLR-11041: MoveReplicaCmd do not specify ulog dir in case of HDFS --- solr/CHANGES.txt | 2 + .../org/apache/solr/cloud/AddReplicaCmd.java | 4 ++ .../org/apache/solr/cloud/MoveReplicaCmd.java | 3 ++ .../handler/admin/CollectionsHandler.java | 2 + .../solr/cloud/MoveReplicaHDFSTest.java | 45 +++++++++++++++++++ .../solrj/request/CollectionAdminRequest.java | 13 ++++++ 6 files changed, 69 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 02d2364d44e..1f357f7552b 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -330,6 +330,8 @@ Bug Fixes * SOLR-11039: Next button in Solr admin UI for collection list pagination does not work. (janhoy) +* SOLR-11041: MoveReplicaCmd do not specify ulog dir in case of HDFS (Cao Manh Dat) + Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java index c42d073b0fb..e9bfebf14b2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/AddReplicaCmd.java @@ -139,6 +139,7 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd { String configName = zkStateReader.readConfigName(collection); String routeKey = message.getStr(ShardParams._ROUTE_); String dataDir = message.getStr(CoreAdminParams.DATA_DIR); + String ulogDir = message.getStr(CoreAdminParams.ULOG_DIR); String instanceDir = message.getStr(CoreAdminParams.INSTANCE_DIR); params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATE.toString()); @@ -161,6 +162,9 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd { if (dataDir != null) { params.set(CoreAdminParams.DATA_DIR, dataDir); } + if (ulogDir != null) { + params.set(CoreAdminParams.ULOG_DIR, ulogDir); + } if (instanceDir != null) { params.set(CoreAdminParams.INSTANCE_DIR, instanceDir); } diff --git a/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java index 4d6e26d67a5..4598f1410f6 100644 --- a/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/MoveReplicaCmd.java @@ -34,6 +34,7 @@ import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; +import org.apache.solr.update.UpdateLog; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -134,11 +135,13 @@ public class MoveReplicaCmd implements Cmd{ return; } + String ulogDir = replica.getStr(CoreAdminParams.ULOG_DIR); ZkNodeProps addReplicasProps = new ZkNodeProps( COLLECTION_PROP, coll.getName(), SHARD_ID_PROP, slice.getName(), CoreAdminParams.NODE, targetNode, CoreAdminParams.NAME, newCoreName, + CoreAdminParams.ULOG_DIR, ulogDir.substring(0, ulogDir.lastIndexOf(UpdateLog.TLOG_NAME)), CoreAdminParams.DATA_DIR, dataDir); if(async!=null) addReplicasProps.getProperties().put(ASYNC, async); NamedList addResult = new NamedList(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 256400e5c43..a4155ea6b4a 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -126,6 +126,7 @@ import static org.apache.solr.common.params.CoreAdminParams.DELETE_DATA_DIR; import static org.apache.solr.common.params.CoreAdminParams.DELETE_INDEX; import static org.apache.solr.common.params.CoreAdminParams.DELETE_INSTANCE_DIR; import static org.apache.solr.common.params.CoreAdminParams.INSTANCE_DIR; +import static org.apache.solr.common.params.CoreAdminParams.ULOG_DIR; import static org.apache.solr.common.params.ShardParams._ROUTE_; import static org.apache.solr.common.util.StrUtils.formatString; @@ -633,6 +634,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission CoreAdminParams.NAME, INSTANCE_DIR, DATA_DIR, + ULOG_DIR, REPLICA_TYPE); return copyPropertiesWithPrefix(req.getParams(), props, COLL_PROP_PREFIX); }), diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java index 367756316da..c350dd5bf61 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java @@ -16,14 +16,24 @@ */ package org.apache.solr.cloud; +import java.io.IOException; + import com.carrotsearch.randomizedtesting.ThreadFilter; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.cloud.hdfs.HdfsTestUtil; +import org.apache.solr.common.cloud.ClusterStateUtil; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.ZkConfigManager; +import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.util.BadHdfsThreadsFilter; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Test; /** * @@ -54,6 +64,41 @@ public class MoveReplicaHDFSTest extends MoveReplicaTest { dfsCluster = null; } + @Test + public void testDataDirAndUlogAreMaintained() throws IOException, SolrServerException { + String coll = "movereplicatest_coll2"; + CollectionAdminRequest.createCollection(coll, "conf1", 1, 1) + .setCreateNodeSet("") + .process(cluster.getSolrClient()); + String hdfsUri = HdfsTestUtil.getURI(dfsCluster); + String dataDir = hdfsUri + "/dummyFolder/dataDir"; + String ulogDir = hdfsUri + "/dummyFolder2/ulogDir"; + CollectionAdminResponse res = CollectionAdminRequest + .addReplicaToShard(coll, "shard1") + .setDataDir(dataDir) + .setUlogDir(ulogDir) + .setNode(cluster.getJettySolrRunner(0).getNodeName()) + .process(cluster.getSolrClient()); + + ulogDir += "/tlog"; + ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader(); + ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000); + DocCollection docCollection = zkStateReader.getClusterState().getCollection(coll); + Replica replica = docCollection.getReplicas().iterator().next(); + assertTrue(replica.getStr("ulogDir"), replica.getStr("ulogDir").equals(ulogDir) || replica.getStr("ulogDir").equals(ulogDir+'/')); + assertTrue(replica.getStr("dataDir"),replica.getStr("dataDir").equals(dataDir) || replica.getStr("dataDir").equals(dataDir+'/')); + + new CollectionAdminRequest.MoveReplica(coll, replica.getName(), cluster.getJettySolrRunner(1).getNodeName()) + .process(cluster.getSolrClient()); + ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 120000); + docCollection = zkStateReader.getClusterState().getCollection(coll); + assertEquals(1, docCollection.getSlice("shard1").getReplicas().size()); + replica = docCollection.getReplicas().iterator().next(); + assertEquals(replica.getNodeName(), cluster.getJettySolrRunner(1).getNodeName()); + assertTrue(replica.getStr("ulogDir"), replica.getStr("ulogDir").equals(ulogDir) || replica.getStr("ulogDir").equals(ulogDir+'/')); + assertTrue(replica.getStr("dataDir"),replica.getStr("dataDir").equals(dataDir) || replica.getStr("dataDir").equals(dataDir+'/')); + + } public static class ForkJoinThreadsFilter implements ThreadFilter { diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index f039ba4048d..88100f31aaf 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -1364,6 +1364,7 @@ public abstract class CollectionAdminRequest protected String routeKey; protected String instanceDir; protected String dataDir; + protected String ulogDir; protected Properties properties; protected Replica.Type type; @@ -1408,6 +1409,10 @@ public abstract class CollectionAdminRequest return instanceDir; } + public String getUlogDir() { + return ulogDir; + } + public AddReplica setInstanceDir(String instanceDir) { this.instanceDir = instanceDir; return this; @@ -1432,6 +1437,11 @@ public abstract class CollectionAdminRequest return this; } + public AddReplica setUlogDir(String ulogDir) { + this.ulogDir = ulogDir; + return this; + } + @Override public SolrParams getParams() { ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); @@ -1452,6 +1462,9 @@ public abstract class CollectionAdminRequest if (dataDir != null) { params.add("dataDir", dataDir); } + if (ulogDir != null) { + params.add("ulogDir", ulogDir); + } if (coreName != null) { params.add("name", coreName); }