From 57f17b111842729552f390dc653ffbaad0b4d658 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Fri, 28 Apr 2017 11:24:53 -0400 Subject: [PATCH] SOLR-9386: Upgrade Zookeeper to 3.4.10 --- lucene/ivy-versions.properties | 2 +- solr/CHANGES.txt | 5 +- .../org/apache/solr/cloud/SolrZkServer.java | 157 +----------------- .../cloud/TestConfigSetsAPIZkFailure.java | 4 +- solr/licenses/zookeeper-3.4.10.jar.sha1 | 1 + solr/licenses/zookeeper-3.4.6.jar.sha1 | 1 - solr/server/solr/zoo.cfg | 14 ++ 7 files changed, 29 insertions(+), 155 deletions(-) create mode 100644 solr/licenses/zookeeper-3.4.10.jar.sha1 delete mode 100644 solr/licenses/zookeeper-3.4.6.jar.sha1 diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index 60abe9a5622..2ef7ad85276 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -186,7 +186,7 @@ org.apache.uima.version = 2.3.1 /org.apache.velocity/velocity = 1.7 /org.apache.velocity/velocity-tools = 2.0 /org.apache.xmlbeans/xmlbeans = 2.6.0 -/org.apache.zookeeper/zookeeper = 3.4.6 +/org.apache.zookeeper/zookeeper = 3.4.10 /org.aspectj/aspectjrt = 1.8.0 org.bouncycastle.version = 1.45 diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 7a59a3e93bd..8f1fe870254 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -139,7 +139,7 @@ Apache Tika 1.13 Carrot2 3.15.0 Velocity 1.7 and Velocity Tools 2.0 Apache UIMA 2.3.1 -Apache ZooKeeper 3.4.6 +Apache ZooKeeper 3.4.10 Jetty 9.3.14.v20161028 Upgrade Notes @@ -150,6 +150,7 @@ Upgrade Notes * JSON Facet API now uses hyper-log-log for numBuckets cardinality calculation and calculates cardinality before filtering buckets by any mincount greater than 1. +* ZooKeeper dependency has been upgraded from 3.4.6 to 3.4.10. Detailed Change List ---------------------- @@ -311,6 +312,8 @@ Other Changes * SOLR-10514: Upgrade Metrics library to 3.2.2. (ab) +* SOLR-9386: Upgrade Zookeeper to 3.4.10. (Shawn Heisey, Steve Rowe) + ================== 6.5.1 ================== Bug Fixes diff --git a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java index 736101add5f..9c2016fe766 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java +++ b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java @@ -22,12 +22,9 @@ import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.quorum.QuorumPeer; import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.apache.zookeeper.server.quorum.QuorumPeerMain; -import org.apache.zookeeper.server.quorum.flexible.QuorumHierarchical; -import org.apache.zookeeper.server.quorum.flexible.QuorumMaj; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -38,8 +35,8 @@ import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.util.Map; -import java.util.Map.Entry; import java.util.Properties; +import java.util.regex.Pattern; public class SolrZkServer { @@ -154,6 +151,7 @@ public class SolrZkServer { // zoo.cfg (which validates that there is a dataDir) class SolrZkServerProps extends QuorumPeerConfig { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final Pattern MISSING_MYID_FILE_PATTERN = Pattern.compile(".*myid file is missing$"); String solrPort; // port that Solr is listening on String zkRun; @@ -312,8 +310,6 @@ class SolrZkServerProps extends QuorumPeerConfig { } } - - // NOTE: copied from ZooKeeper 3.2 /** * Parse config from a Properties. * @param zkProp Properties to parse from. @@ -321,157 +317,18 @@ class SolrZkServerProps extends QuorumPeerConfig { @Override public void parseProperties(Properties zkProp) throws IOException, ConfigException { - for (Entry entry : zkProp.entrySet()) { - String key = entry.getKey().toString().trim(); - String value = entry.getValue().toString().trim(); - if (key.equals("dataDir")) { - dataDir = value; - } else if (key.equals("dataLogDir")) { - dataLogDir = value; - } else if (key.equals("clientPort")) { - setClientPort(Integer.parseInt(value)); - } else if (key.equals("tickTime")) { - tickTime = Integer.parseInt(value); - } else if (key.equals("initLimit")) { - initLimit = Integer.parseInt(value); - } else if (key.equals("syncLimit")) { - syncLimit = Integer.parseInt(value); - } else if (key.equals("electionAlg")) { - electionAlg = Integer.parseInt(value); - } else if (key.equals("maxClientCnxns")) { - maxClientCnxns = Integer.parseInt(value); - } else if (key.startsWith("server.")) { - int dot = key.indexOf('.'); - long sid = Long.parseLong(key.substring(dot + 1)); - String parts[] = value.split(":"); - if ((parts.length != 2) && (parts.length != 3)) { - LOG.error(value - + " does not have the form host:port or host:port:port"); - } - InetSocketAddress addr = new InetSocketAddress(parts[0], - Integer.parseInt(parts[1])); - if (parts.length == 2) { - servers.put(Long.valueOf(sid), new QuorumPeer.QuorumServer(sid, addr)); - } else if (parts.length == 3) { - InetSocketAddress electionAddr = new InetSocketAddress( - parts[0], Integer.parseInt(parts[2])); - servers.put(Long.valueOf(sid), new QuorumPeer.QuorumServer(sid, addr, - electionAddr)); - } - } else if (key.startsWith("group")) { - int dot = key.indexOf('.'); - long gid = Long.parseLong(key.substring(dot + 1)); - - numGroups++; - - String parts[] = value.split(":"); - for(String s : parts){ - long sid = Long.parseLong(s); - if(serverGroup.containsKey(sid)) - throw new ConfigException("Server " + sid + "is in multiple groups"); - else - serverGroup.put(sid, gid); - } - - } else if(key.startsWith("weight")) { - int dot = key.indexOf('.'); - long sid = Long.parseLong(key.substring(dot + 1)); - serverWeight.put(sid, Long.parseLong(value)); - } else { - System.setProperty("zookeeper." + key, value); - } - } - if (dataDir == null) { - throw new IllegalArgumentException("dataDir is not set"); - } - if (dataLogDir == null) { - dataLogDir = dataDir; - } else { - if (!new File(dataLogDir).isDirectory()) { - throw new IllegalArgumentException("dataLogDir " + dataLogDir - + " is missing."); - } - } - - if (tickTime == 0) { - throw new IllegalArgumentException("tickTime is not set"); - } - if (servers.size() > 1) { - if (initLimit == 0) { - throw new IllegalArgumentException("initLimit is not set"); - } - if (syncLimit == 0) { - throw new IllegalArgumentException("syncLimit is not set"); - } - /* - * If using FLE, then every server requires a separate election - * port. - */ - if (electionAlg != 0) { - for (QuorumPeer.QuorumServer s : servers.values()) { - if (s.electionAddr == null) - throw new IllegalArgumentException( - "Missing election port for server: " + s.id); - } - } - - /* - * Default of quorum config is majority - */ - if(serverGroup.size() > 0){ - if(servers.size() != serverGroup.size()) - throw new ConfigException("Every server must be in exactly one group"); - /* - * The deafult weight of a server is 1 - */ - for(QuorumPeer.QuorumServer s : servers.values()){ - if(!serverWeight.containsKey(s.id)) - serverWeight.put(s.id, (long) 1); - } - - /* - * Set the quorumVerifier to be QuorumHierarchical - */ - quorumVerifier = new QuorumHierarchical(numGroups, - serverWeight, serverGroup); - } else { - /* - * The default QuorumVerifier is QuorumMaj - */ - - LOG.info("Defaulting to majority quorums"); - quorumVerifier = new QuorumMaj(servers.size()); - } - - File myIdFile = new File(dataDir, "myid"); - if (!myIdFile.exists()) { - ///////////////// ADDED FOR SOLR ////// + try { + super.parseProperties(zkProp); + } catch (IllegalArgumentException e) { + if (MISSING_MYID_FILE_PATTERN.matcher(e.getMessage()).matches()) { Long myid = getMyServerId(); if (myid != null) { serverId = myid; return; } if (zkRun == null) return; - //////////////// END ADDED FOR SOLR ////// - throw new IllegalArgumentException(myIdFile.toString() - + " file is missing"); - } - - BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(myIdFile), StandardCharsets.UTF_8)); - String myIdString; - try { - myIdString = br.readLine(); - } finally { - br.close(); - } - try { - serverId = Long.parseLong(myIdString); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("serverid " + myIdString - + " is not a number"); } + throw e; } } - - } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java index bcb0a1bd8db..34acb2509e0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java @@ -290,8 +290,8 @@ public class TestConfigSetsAPIZkFailure extends SolrTestCaseJ4 { } @Override - public List convertLong(Long aclL) { - return zkdb.convertLong(aclL); + public List aclForNode(DataNode n) { + return zkdb.aclForNode(n); } @Override diff --git a/solr/licenses/zookeeper-3.4.10.jar.sha1 b/solr/licenses/zookeeper-3.4.10.jar.sha1 new file mode 100644 index 00000000000..9b94ebdde40 --- /dev/null +++ b/solr/licenses/zookeeper-3.4.10.jar.sha1 @@ -0,0 +1 @@ +08eebdbb7a9df83e02eaa42d0e5da0b57bf2e4da diff --git a/solr/licenses/zookeeper-3.4.6.jar.sha1 b/solr/licenses/zookeeper-3.4.6.jar.sha1 deleted file mode 100644 index f64d0949c7b..00000000000 --- a/solr/licenses/zookeeper-3.4.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -01b2502e29da1ebaade2357cd1de35a855fa3755 diff --git a/solr/server/solr/zoo.cfg b/solr/server/solr/zoo.cfg index aea451885e4..7e42d8c8465 100644 --- a/solr/server/solr/zoo.cfg +++ b/solr/server/solr/zoo.cfg @@ -15,3 +15,17 @@ syncLimit=5 # clientPort=2181 # NOTE: Solr sets this based on zkRun / zkHost params +# the maximum number of client connections. +# increase this if you need to handle more clients +#maxClientCnxns=60 +# +# Be sure to read the maintenance section of the +# administrator guide before turning on autopurge. +# +# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance +# +# The number of snapshots to retain in dataDir +#autopurge.snapRetainCount=3 +# Purge task interval in hours +# Set to "0" to disable auto purge feature +#autopurge.purgeInterval=1