From 911ee1dcd3435269e6f3970d3754aee4ab91e725 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Fri, 22 Nov 2013 05:05:36 +0000 Subject: [PATCH] SOLR-5308: Use source collection's configName to create temp collection. Fixed NPE routing rule wait loop. Wait for temp collection leader to be active before splitting the source index. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1544414 13f79535-47bb-0310-9956-ffa450edef68 --- .../cloud/OverseerCollectionProcessor.java | 27 +++++++++++++--- .../org/apache/solr/cloud/ZkController.java | 29 ----------------- .../org/apache/solr/core/CoreContainer.java | 2 +- .../org/apache/solr/core/ZkContainer.java | 2 +- .../component/QueryElevationComponent.java | 2 +- .../apache/solr/cloud/ZkControllerTest.java | 2 +- .../solr/common/cloud/ZkStateReader.java | 32 +++++++++++++++++++ 7 files changed, 58 insertions(+), 38 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java index c149a94a433..c0c7a93601b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java @@ -1163,7 +1163,7 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread { Map rules = zkStateReader.getClusterState().getSlice(sourceCollection.getName(), sourceSlice.getName()).getRoutingRules(); if (rules != null) { RoutingRule rule = rules.get(splitKey); - if (rule.getRouteRanges().contains(splitRange)) { + if (rule != null && rule.getRouteRanges().contains(splitRange)) { added = true; break; } @@ -1179,13 +1179,13 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread { Replica sourceLeader = sourceSlice.getLeader(); // create a temporary collection with just one node on the shard leader - String sourceLeaderUrl = zkStateReader.getZkClient().getBaseUrlForNodeName(sourceLeader.getNodeName()); - if (sourceLeaderUrl.startsWith("http://")) sourceLeaderUrl = sourceLeaderUrl.substring(7); + String configName = zkStateReader.readConfigName(sourceCollection.getName()); Map props = ZkNodeProps.makeMap( QUEUE_OPERATION, CREATECOLLECTION, "name", tempSourceCollectionName, REPLICATION_FACTOR, 1, NUM_SLICES, 1, + COLL_CONF, configName, CREATE_NODE_SET, sourceLeader.getNodeName()); log.info("Creating temporary collection: " + props); createCollection(clusterState, new ZkNodeProps(props), results); @@ -1194,6 +1194,23 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread { Slice tempSourceSlice = clusterState.getCollection(tempSourceCollectionName).getSlices().iterator().next(); Replica tempSourceLeader = clusterState.getLeader(tempSourceCollectionName, tempSourceSlice.getName()); + String tempCollectionReplica1 = tempSourceCollectionName + "_" + tempSourceSlice.getName() + "_replica1"; + String coreNodeName = waitForCoreNodeName(clusterState.getCollection(tempSourceCollectionName), + zkStateReader.getZkClient().getBaseUrlForNodeName(sourceLeader.getNodeName()), tempCollectionReplica1); + // wait for the replicas to be seen as active on temp source leader + log.info("Asking source leader to wait for: " + tempCollectionReplica1 + " to be alive on: " + sourceLeader.getNodeName()); + CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState(); + cmd.setCoreName(tempCollectionReplica1); + cmd.setNodeName(sourceLeader.getNodeName()); + cmd.setCoreNodeName(coreNodeName); + cmd.setState(ZkStateReader.ACTIVE); + cmd.setCheckLive(true); + cmd.setOnlyIfLeader(true); + sendShardRequest(tempSourceLeader.getNodeName(), new ModifiableSolrParams(cmd.getParams())); + + collectShardResponses(results, true, + "MIGRATE failed to create temp collection leader or timed out waiting for it to come up"); + log.info("Asking source leader to split index"); params = new ModifiableSolrParams(); params.set(CoreAdminParams.ACTION, CoreAdminAction.SPLIT.toString()); @@ -1215,11 +1232,11 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread { params.set(CoreAdminParams.SHARD, tempSourceSlice.getName()); sendShardRequest(targetLeader.getNodeName(), params); - String coreNodeName = waitForCoreNodeName(clusterState.getCollection(tempSourceCollectionName), + coreNodeName = waitForCoreNodeName(clusterState.getCollection(tempSourceCollectionName), zkStateReader.getZkClient().getBaseUrlForNodeName(targetLeader.getNodeName()), tempCollectionReplica2); // wait for the replicas to be seen as active on temp source leader log.info("Asking temp source leader to wait for: " + tempCollectionReplica2 + " to be alive on: " + targetLeader.getNodeName()); - CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState(); + cmd = new CoreAdminRequest.WaitForState(); cmd.setCoreName(tempSourceLeader.getStr("core")); cmd.setNodeName(targetLeader.getNodeName()); cmd.setCoreNodeName(coreNodeName); diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 8b3d99cf594..ee2a8e2956a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -733,35 +733,6 @@ public final class ZkController { return zkClient.exists(path, true); } - /** - * Returns config value - */ - public String readConfigName(String collection) throws KeeperException, - InterruptedException { - - String configName = null; - - String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection; - if (log.isInfoEnabled()) { - log.info("Load collection config from:" + path); - } - byte[] data = zkClient.getData(path, null, null, true); - - if(data != null) { - ZkNodeProps props = ZkNodeProps.load(data); - configName = props.getStr(CONFIGNAME_PROP); - } - - if (configName != null && !zkClient.exists(CONFIGS_ZKNODE + "/" + configName, true)) { - log.error("Specified config does not exist in ZooKeeper:" + configName); - throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, - "Specified config does not exist in ZooKeeper:" + configName); - } - - return configName; - } - - /** * Register shard with ZooKeeper. diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index fe3c455be23..9aaabfb7591 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -676,7 +676,7 @@ public class CoreContainer { String collection = cd.getCloudDescriptor().getCollectionName(); zkSys.getZkController().createCollectionZkNode(cd.getCloudDescriptor()); - String zkConfigName = zkSys.getZkController().readConfigName(collection); + String zkConfigName = zkSys.getZkController().getZkStateReader().readConfigName(collection); if (zkConfigName == null) { log.error("Could not find config name for collection:" + collection); throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, diff --git a/solr/core/src/java/org/apache/solr/core/ZkContainer.java b/solr/core/src/java/org/apache/solr/core/ZkContainer.java index ab349612045..e61487aebcc 100644 --- a/solr/core/src/java/org/apache/solr/core/ZkContainer.java +++ b/solr/core/src/java/org/apache/solr/core/ZkContainer.java @@ -220,7 +220,7 @@ public class ZkContainer { String collection = dcore.getCloudDescriptor().getCollectionName(); zkController.createCollectionZkNode(dcore.getCloudDescriptor()); - zkConfigName = zkController.readConfigName(collection); + zkConfigName = zkController.getZkStateReader().readConfigName(collection); if (zkConfigName == null) { log.error("Could not find config name for collection:" + collection); throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java index 476e53d66b9..b0caa898c28 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java @@ -208,7 +208,7 @@ public class QueryElevationComponent extends SearchComponent implements SolrCore ZkController zkController = core.getCoreDescriptor().getCoreContainer().getZkController(); if (zkController != null) { // TODO : shouldn't have to keep reading the config name when it has been read before - exists = zkController.configFileExists(zkController.readConfigName(core.getCoreDescriptor().getCloudDescriptor().getCollectionName()), f); + exists = zkController.configFileExists(zkController.getZkStateReader().readConfigName(core.getCoreDescriptor().getCloudDescriptor().getCollectionName()), f); } else { File fC = new File(core.getResourceLoader().getConfigDir(), f); File fD = new File(core.getDataDir(), f); diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java index 7343cc49363..3352e22ab86 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java @@ -188,7 +188,7 @@ public class ZkControllerTest extends SolrTestCaseJ4 { } }); try { - String configName = zkController.readConfigName(COLLECTION_NAME); + String configName = zkController.getZkStateReader().readConfigName(COLLECTION_NAME); assertEquals(configName, actualConfigName); } finally { zkController.close(); diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java index 86928150c0a..35bc01e7d04 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -74,6 +74,9 @@ public class ZkStateReader { public static final String ACTIVE = "active"; public static final String DOWN = "down"; public static final String SYNC = "sync"; + + public static final String CONFIGS_ZKNODE = "/configs"; + public final static String CONFIGNAME_PROP="configName"; private volatile ClusterState clusterState; @@ -115,6 +118,35 @@ public class ZkStateReader { } } + /** + * Returns config value + * @param collection + */ + public String readConfigName(String collection) throws KeeperException, + InterruptedException { + + String configName = null; + + String path = COLLECTIONS_ZKNODE + "/" + collection; + if (log.isInfoEnabled()) { + log.info("Load collection config from:" + path); + } + byte[] data = zkClient.getData(path, null, null, true); + + if(data != null) { + ZkNodeProps props = ZkNodeProps.load(data); + configName = props.getStr(CONFIGNAME_PROP); + } + + if (configName != null && !zkClient.exists(CONFIGS_ZKNODE + "/" + configName, true)) { + log.error("Specified config does not exist in ZooKeeper:" + configName); + throw new ZooKeeperException(ErrorCode.SERVER_ERROR, + "Specified config does not exist in ZooKeeper:" + configName); + } + + return configName; + } + private static class ZKTF implements ThreadFactory { private static ThreadGroup tg = new ThreadGroup("ZkStateReader");