Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-7176: zkcli script can perfrom the CLUSTERPROP command without a running Solr cluster 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1675331 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Noble Paul 2015-04-22 11:12:33 +00:00
parent 5afa59c004
commit 8c5fcd1ff6
8 changed files with 128 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -108,6 +108,9 @@ New Features
top_authors : { terms : { field:author, limit:5 } }
(yonik)
* SOLR-7176: zkcli script can perfrom the CLUSTERPROP command without a running Solr cluster
(Hrishikesh Gadre, Per Steffensen, Noble Paul)
Bug Fixes
----------------------

24
solr/core/src/java/org/apache/solr/cloud/Overseer.java
View File

@ -100,7 +100,6 @@ public class Overseer implements Closeable {
private final Stats zkStats;
private Map clusterProps;
private boolean isClosed = false;
public ClusterStateUpdater(final ZkStateReader reader, final String myId, Stats zkStats) {
@ -113,7 +112,6 @@ public class Overseer implements Closeable {
this.completedMap = getCompletedMap(zkClient);
this.myId = myId;
this.reader = reader;
clusterProps = reader.getClusterProps();
}
public Stats getStateUpdateQueueStats() {
@ -344,9 +342,6 @@ public class Overseer implements Closeable {
return new CollectionMutator(getZkStateReader()).deleteShard(clusterState, message);
case ADDREPLICA:
return new SliceMutator(getZkStateReader()).addReplica(clusterState, message);
case CLUSTERPROP:
handleProp(message);
break;
case ADDREPLICAPROP:
return new ReplicaMutator(getZkStateReader()).addReplicaProperty(clusterState, message);
case DELETEREPLICAPROP:
@ -397,25 +392,6 @@ public class Overseer implements Closeable {
return ZkStateWriter.NO_OP;
}
private void handleProp(ZkNodeProps message) {
String name = message.getStr(NAME);
String val = message.getStr("val");
Map m = reader.getClusterProps();
if(val ==null) m.remove(name);
else m.put(name,val);
try {
if (reader.getZkClient().exists(ZkStateReader.CLUSTER_PROPS, true))
reader.getZkClient().setData(ZkStateReader.CLUSTER_PROPS, ZkStateReader.toJSON(m), true);
else
reader.getZkClient().create(ZkStateReader.CLUSTER_PROPS, ZkStateReader.toJSON(m),CreateMode.PERSISTENT, true);
clusterProps = reader.getClusterProps();
} catch (Exception e) {
log.error("Unable to set cluster property", e);
}
}
private LeaderStatus amILeader() {
TimerContext timerContext = stats.time("am_i_leader");
boolean success = true;

3
solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
View File

@ -130,9 +130,6 @@ public class OverseerCollectionProcessor implements Runnable, Closeable {
public int maxParallelThreads = 10;
public static final Set<String> KNOWN_CLUSTER_PROPS = ImmutableSet.of(ZkStateReader.LEGACY_CLOUD, ZkStateReader.URL_SCHEME,
ZkStateReader.AUTO_ADD_REPLICAS);
public static final Map<String,Object> COLL_PROPS = ZkNodeProps.makeMap(
ROUTER, DocRouter.DEFAULT_NAME,
ZkStateReader.REPLICATION_FACTOR, "1",

43
solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
View File

@ -1,5 +1,8 @@
package org.apache.solr.cloud;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonParams.*;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
@ -10,15 +13,19 @@ import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.OnReconnect;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkConfigManager;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.core.CoreContainer;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@ -123,7 +130,9 @@ public class ZkCLI {
"run zk internally by passing the solr run port - only for clusters on one machine (tests, dev)");
options.addOption("h", HELP, false, "bring up this help page");
options.addOption(NAME, true, "name of the cluster property to set");
options.addOption(VALUE_LONG, true, "value of the cluster to set");
try {
// parse the command line arguments
CommandLine line = parser.parse(options, args);
@ -145,6 +154,7 @@ public class ZkCLI {
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + GET_FILE + " /solr.xml solr.xml.file");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLEAR + " /solr");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LIST);
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLUSTERPROP + " -" + NAME + " urlScheme -" + VALUE_LONG + " https" );
return;
}
@ -294,6 +304,37 @@ public class ZkCLI {
}
byte [] data = zkClient.getData(arglist.get(0).toString(), null, null, true);
FileUtils.writeByteArrayToFile(new File(arglist.get(1).toString()), data);
} else if (CollectionAction.get(line.getOptionValue(CMD)) == CLUSTERPROP) {
if(!line.hasOption(NAME)) {
System.out.println("-" + NAME + " is required for " + CLUSTERPROP);
}
String propertyName = line.getOptionValue(NAME);
//If -val option is missing, we will use the null value. This is required to maintain
//compatibility with Collections API.
String propertyValue = line.getOptionValue(VALUE_LONG);
ZkStateReader reader = new ZkStateReader(zkClient);
try {
reader.setClusterProperty(propertyName, propertyValue);
} catch (SolrException ex) {
//This can happen if two concurrent invocations of this command collide
//with each other. Here we are just adding a defensive check to see if
//the value is already set to expected value. If yes, then we don't
//fail the command.
Throwable cause = ex.getCause();
if(cause instanceof KeeperException.NodeExistsException
|| cause instanceof KeeperException.BadVersionException) {
String currentValue = (String)reader.getClusterProps().get(propertyName);
if((currentValue == propertyValue) || (currentValue != null && currentValue.equals(propertyValue))) {
return;
}
}
System.out.println("Unable to set the cluster property due to following error : " +
ex.getLocalizedMessage() +
((cause instanceof KeeperException.BadVersionException)?". Try again":""));
System.exit(1);
} finally {
reader.close();
}
}
} finally {
if (solrPort != null) {

14
solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
View File

@ -20,7 +20,6 @@ package org.apache.solr.handler.admin;
import static org.apache.solr.cloud.Overseer.*;
import static org.apache.solr.cloud.OverseerCollectionProcessor.*;
import static org.apache.solr.common.cloud.DocCollection.*;
import static org.apache.solr.common.cloud.ZkNodeProps.*;
import static org.apache.solr.common.cloud.ZkStateReader.*;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonParams.*;
@ -569,17 +568,8 @@ public class CollectionsHandler extends RequestHandlerBase {
private void handleProp(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
req.getParams().required().check(NAME);
String name = req.getParams().get(NAME);
if(!OverseerCollectionProcessor.KNOWN_CLUSTER_PROPS.contains(name)){
throw new SolrException(ErrorCode.BAD_REQUEST, "Not a known cluster property "+ name);
}
Map<String,Object> props = ZkNodeProps.makeMap(
Overseer.QUEUE_OPERATION, CLUSTERPROP.toLower() );
copyIfNotNull(req.getParams(),props,
NAME,
"val");
Overseer.getInQueue(coreContainer.getZkController().getZkClient()).offer(ZkStateReader.toJSON(props)) ;
String val = req.getParams().get(VALUE_LONG);
coreContainer.getZkController().getZkStateReader().setClusterProperty(name, val);
}
static Set<String> KNOWN_ROLES = ImmutableSet.of("overseer");

20
solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
View File

@ -298,6 +298,26 @@ public class ZkCLITest extends SolrTestCaseJ4 {
zkClient.close();
}
@Test
public void testSetClusterProperty() throws Exception {
ZkStateReader reader = new ZkStateReader(zkClient);
try {
// add property urlScheme=http
String[] args = new String[] {"-zkhost", zkServer.getZkAddress(),
"-cmd", "CLUSTERPROP", "-name", "urlScheme", "-val", "http"};
ZkCLI.main(args);
assertEquals("http", reader.getClusterProps().get("urlScheme"));
// remove it again
args = new String[] {"-zkhost", zkServer.getZkAddress(),
"-cmd", "CLUSTERPROP", "-name", "urlScheme"};
ZkCLI.main(args);
assertNull(reader.getClusterProps().get("urlScheme"));
} finally {
reader.close();
}
}
@Override
public void tearDown() throws Exception {
if (VERBOSE) {

61
solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
View File

@ -20,6 +20,7 @@ package org.apache.solr.common.cloud;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.ByteUtils;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
@ -51,6 +52,9 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import static java.util.Arrays.asList;
import static java.util.Collections.unmodifiableSet;
public class ZkStateReader implements Closeable {
private static Logger log = LoggerFactory.getLogger(ZkStateReader.class);
@ -113,6 +117,10 @@ public class ZkStateReader implements Closeable {
private final ZkConfigManager configManager;
public static final Set<String> KNOWN_CLUSTER_PROPS = unmodifiableSet(new HashSet<>(asList(
LEGACY_CLOUD,
URL_SCHEME,
AUTO_ADD_REPLICAS)));
//
// convenience methods... should these go somewhere else?
@ -765,7 +773,58 @@ public class ZkStateReader implements Closeable {
throw new SolrException(ErrorCode.SERVER_ERROR,"Error reading cluster properties",e) ;
}
}
/**
* This method sets a cluster property.
*
* @param propertyName The property name to be set.
* @param propertyValue The value of the property.
*/
public void setClusterProperty(String propertyName, String propertyValue) {
if (!KNOWN_CLUSTER_PROPS.contains(propertyName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Not a known cluster property " + propertyName);
}
for (; ; ) {
Stat s = new Stat();
try {
if (getZkClient().exists(CLUSTER_PROPS, true)) {
int v = 0;
Map properties = (Map) fromJSON(getZkClient().getData(CLUSTER_PROPS, null, s, true));
if (propertyValue == null) {
//Don't update ZK unless absolutely necessary.
if (properties.get(propertyName) != null) {
properties.remove(propertyName);
getZkClient().setData(CLUSTER_PROPS, toJSON(properties), s.getVersion(), true);
}
} else {
//Don't update ZK unless absolutely necessary.
if (!propertyValue.equals(properties.get(propertyName))) {
properties.put(propertyName, propertyValue);
getZkClient().setData(CLUSTER_PROPS, toJSON(properties), s.getVersion(), true);
}
}
} else {
Map properties = new LinkedHashMap();
properties.put(propertyName, propertyValue);
getZkClient().create(CLUSTER_PROPS, toJSON(properties), CreateMode.PERSISTENT, true);
}
} catch (KeeperException.BadVersionException bve) {
log.warn("Race condition while trying to set a new cluster prop on current version " + s.getVersion());
//race condition
continue;
} catch (KeeperException.NodeExistsException nee) {
log.warn("Race condition while trying to set a new cluster prop on current version " + s.getVersion());
//race condition
continue;
} catch (Exception ex) {
log.error("Error updating path " + CLUSTER_PROPS, ex);
throw new SolrException(ErrorCode.SERVER_ERROR, "Error updating cluster property " + propertyName, ex);
}
break;
}
}
/**
* Returns the baseURL corresponding to a given node's nodeName --
* NOTE: does not (currently) imply that the nodeName (or resulting

1
solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
View File

@ -232,5 +232,6 @@ public interface CommonParams {
public static final String PATH = "path";
public static final String NAME = "name";
public static final String VALUE_LONG = "val";
}