From 1e42240e8969342cb4ccceb007d1aa4031239b01 Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Sun, 1 Dec 2013 21:58:45 +0000 Subject: [PATCH] SOLR-5482: add option to simulate hard jetty fails with iptables git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1546863 13f79535-47bb-0310-9956-ffa450edef68 --- .../solr/common/cloud/SolrZooKeeper.java | 5 + .../java/org/apache/solr/SolrTestCaseJ4.java | 3 + .../cloud/AbstractFullDistribZkTestBase.java | 2 +- .../org/apache/solr/cloud/ChaosMonkey.java | 33 ++++- .../java/org/apache/solr/cloud/IpTables.java | 116 ++++++++++++++++++ 5 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/IpTables.java diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZooKeeper.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZooKeeper.java index 294dba80654..df352cdd7cf 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZooKeeper.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZooKeeper.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.net.SocketAddress; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; @@ -45,6 +46,10 @@ public class SolrZooKeeper extends ZooKeeper { return cnxn; } + public SocketAddress getSocketAddress() { + return testableLocalSocketAddress(); + } + /** * Cause this ZooKeeper object to stop receiving from the ZooKeeperServer * for the given number of milliseconds. diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index 61f2d1be823..f4d48e4c125 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -47,6 +47,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.QuickPatchThreadsFilter; import org.apache.lucene.util._TestUtil; import org.apache.solr.client.solrj.util.ClientUtils; +import org.apache.solr.cloud.IpTables; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; @@ -143,6 +144,8 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { System.clearProperty("tests.shardhandler.randomSeed"); System.clearProperty("enable.update.log"); System.clearProperty("useCompoundFile"); + + IpTables.unblockAllPorts(); } private static boolean changedFactory = false; diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java index db92d136eb4..9fd762a0a79 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java @@ -210,7 +210,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes } @AfterClass - public static void afterClass() { + public static void afterClass() throws Exception { System.clearProperty("solrcloud.update.delay"); System.clearProperty("genericCoreNodeNames"); } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java index c1128f9bdf0..3e6091740da 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/ChaosMonkey.java @@ -17,12 +17,15 @@ package org.apache.solr.cloud; * limitations under the License. */ +import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; +import javax.servlet.Filter; + import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.embedded.JettySolrRunner; @@ -226,6 +229,21 @@ public class ChaosMonkey { } public static void kill(CloudJettyRunner cjetty) throws Exception { + FilterHolder filterHolder = cjetty.jetty.getDispatchFilter(); + if (filterHolder != null) { + Filter filter = filterHolder.getFilter(); + if (filter != null) { + CoreContainer cores = ((SolrDispatchFilter) filter).getCores(); + if (cores != null) { + int zklocalport = ((InetSocketAddress) cores.getZkController() + .getZkClient().getSolrZooKeeper().getSocketAddress()).getPort(); + IpTables.blockPort(zklocalport); + } + } + } + + IpTables.blockPort(cjetty.jetty.getLocalPort()); + JettySolrRunner jetty = cjetty.jetty; monkeyLog("kill shard! " + jetty.getLocalPort()); @@ -534,7 +552,8 @@ public class ChaosMonkey { } public static boolean start(JettySolrRunner jetty) throws Exception { - + + IpTables.unblockPort(jetty.getLocalPort()); try { jetty.start(); } catch (Exception e) { @@ -555,6 +574,18 @@ public class ChaosMonkey { } } } + FilterHolder filterHolder = jetty.getDispatchFilter(); + if (filterHolder != null) { + Filter filter = filterHolder.getFilter(); + if (filter != null) { + CoreContainer cores = ((SolrDispatchFilter) filter).getCores(); + if (cores != null) { + int zklocalport = ((InetSocketAddress) cores.getZkController() + .getZkClient().getSolrZooKeeper().getSocketAddress()).getPort(); + IpTables.unblockPort(zklocalport); + } + } + } return true; } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/IpTables.java b/solr/test-framework/src/java/org/apache/solr/cloud/IpTables.java new file mode 100644 index 00000000000..2bdd14d821e --- /dev/null +++ b/solr/test-framework/src/java/org/apache/solr/cloud/IpTables.java @@ -0,0 +1,116 @@ +package org.apache.solr.cloud; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * To use, tests must be able to run iptables, eg sudo chmod u+s iptables + */ +public class IpTables { + static final Logger log = LoggerFactory + .getLogger(IpTables.class); + + private static boolean ENABLED = Boolean.getBoolean("solr.tests.use.iptables"); + static class ThreadPumper { + + public ThreadPumper() {} + + public static Thread start(final InputStream from, final OutputStream to, final boolean verbose) { + Thread t = new Thread() { + @Override + public void run() { + try { + byte [] buffer = new byte [1024]; + int len; + while ((len = from.read(buffer)) != -1) { + if (verbose) { + to.write(buffer, 0, len); + } + } + } catch (IOException e) { + System.err.println("Couldn't pipe from the forked process: " + e.toString()); + } + } + }; + t.start(); + return t; + } + } + + private static Set BLOCK_PORTS = Collections.synchronizedSet(new HashSet()); + + public static void blockPort(int port) throws IOException, + InterruptedException { + if (ENABLED) { + log.info("Block port with iptables: " + port); + BLOCK_PORTS.add(port); + runCmd(("iptables -A INPUT -p tcp --dport " + port + " -j DROP") + .split("\\s")); + runCmd(("iptables -A OUTPUT -p tcp --dport " + port + " -j DROP") + .split("\\s")); + } + } + + public static void unblockPort(int port) throws IOException, + InterruptedException { + if (ENABLED) { + log.info("Unblock port with iptables: " + port); + runCmd(("iptables -D INPUT -p tcp --dport " + port + " -j DROP") + .split("\\s")); + runCmd(("iptables -D OUTPUT -p tcp --dport " + port + " -j DROP") + .split("\\s")); + } + } + + public static void unblockAllPorts() throws IOException, InterruptedException { + if (ENABLED) { + log.info("Unblocking any ports previously blocked with iptables..."); + for (Integer port : BLOCK_PORTS) { + IpTables.unblockPort(port); + } + } + } + + private static void runCmd(String[] cmd) throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder(cmd); + + pb.redirectErrorStream(true); + Process p = pb.start(); + + // We pump everything to stderr. + PrintStream childOut = System.err; + Thread stdoutPumper = ThreadPumper.start(p.getInputStream(), childOut, true); + Thread stderrPumper = ThreadPumper.start(p.getErrorStream(), childOut, true); + if (true) childOut.println(">>> Begin subprocess output"); + p.waitFor(); + stdoutPumper.join(); + stderrPumper.join(); + if (true) childOut.println("<<< End subprocess output"); + } +}