From 9842f45a49afb81ff81d16fcb51bd0c0bb717b62 Mon Sep 17 00:00:00 2001 From: Clebert Suconic Date: Mon, 3 Aug 2020 09:12:44 -0400 Subject: [PATCH] ARTEMIS-2867 Do not cache IPs on DNS Entries for NetworkHealthCheck In case of a DNS outage, the pinger should still fail If we cache the InetAddress this would not be possible --- .../core/server/NetworkHealthCheck.java | 66 ++++--- .../artemis/utils/NetworkHealthTest.java | 3 +- .../artemis/utils/network/NetUtil.java | 8 +- .../core/management/impl/AbstractControl.java | 2 +- .../failover/NetworkIsolationTest.java | 11 +- tests/smoke-tests/pom.xml | 42 +++++ .../broker.xml | 139 +++++++++++++++ .../management.xml | 20 +++ .../dnsswitch-replicated-backup/broker.xml | 6 +- .../broker.xml | 4 +- .../broker.xml | 139 +++++++++++++++ .../management.xml | 20 +++ .../dnsswitch-replicated-main/broker.xml | 4 +- .../tests/smoke/dnsswitch/DNSSwitchTest.java | 165 ++++++++++++------ 14 files changed, 536 insertions(+), 93 deletions(-) create mode 100644 tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/broker.xml create mode 100644 tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/management.xml create mode 100644 tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/broker.xml create mode 100644 tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/management.xml diff --git a/artemis-commons/src/main/java/org/apache/activemq/artemis/core/server/NetworkHealthCheck.java b/artemis-commons/src/main/java/org/apache/activemq/artemis/core/server/NetworkHealthCheck.java index 5c8fc823bd..559ca17e36 100644 --- a/artemis-commons/src/main/java/org/apache/activemq/artemis/core/server/NetworkHealthCheck.java +++ b/artemis-commons/src/main/java/org/apache/activemq/artemis/core/server/NetworkHealthCheck.java @@ -46,7 +46,7 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent { private static final Logger logger = Logger.getLogger(NetworkHealthCheck.class); private final Set componentList = new ConcurrentHashSet<>(); - private final Set addresses = new ConcurrentHashSet<>(); + private final Set addresses = new ConcurrentHashSet<>(); private final Set urls = new ConcurrentHashSet<>(); private NetworkInterface networkInterface; @@ -104,7 +104,7 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent { return this; } - public Set getAddresses() { + public Set getAddresses() { return addresses; } @@ -127,7 +127,8 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent { for (String address : addresses) { if (!address.trim().isEmpty()) { try { - this.addAddress(InetAddress.getByName(address.trim())); + String strAddress = address.trim(); + this.addAddress(strAddress); } catch (Exception e) { ActiveMQUtilLogger.LOGGER.failedToParseAddressList(e, addressList); } @@ -204,22 +205,23 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent { return this; } - public NetworkHealthCheck addAddress(InetAddress address) { - if (!check(address)) { - ActiveMQUtilLogger.LOGGER.addressWasntReacheable(address.toString()); + public NetworkHealthCheck addAddress(String straddress) { + InetAddress address = internalCheck(straddress); + if (address == null) { + ActiveMQUtilLogger.LOGGER.addressWasntReacheable(straddress); } - if (!ignoreLoopback && address.isLoopbackAddress()) { - ActiveMQUtilLogger.LOGGER.addressloopback(address.toString()); + if (!ignoreLoopback && address != null && address.isLoopbackAddress()) { + ActiveMQUtilLogger.LOGGER.addressloopback(straddress); } else { - addresses.add(address); + addresses.add(straddress); checkStart(); } return this; } - public NetworkHealthCheck removeAddress(InetAddress address) { - addresses.remove(address); + public NetworkHealthCheck removeAddress(String straddress) { + addresses.remove(straddress); return this; } @@ -267,7 +269,11 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent { private void checkStart() { if (!isStarted() && (!addresses.isEmpty() || !urls.isEmpty()) && !componentList.isEmpty()) { - start(); + try { + this.run(); // run the first check immediately, this is to immediately shutdown the server if there's no net + } finally { + start(); + } } } @@ -311,7 +317,7 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent { return true; } - for (InetAddress address : addresses) { + for (String address : addresses) { if (check(address)) { return true; } @@ -326,23 +332,37 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent { return false; } - public boolean check(InetAddress address) { - if (address == null) { + public boolean check(String straddress) { + if (straddress == null) { return false; } + return internalCheck(straddress) != null; + } + + private InetAddress internalCheck(String straddress) { try { - if (!hasCustomPingCommand() && isReachable(address)) { - if (logger.isTraceEnabled()) { - logger.tracef(address + " OK"); - } - return true; + InetAddress address = InetAddress.getByName(straddress); + address = InetAddress.getByName(address.getHostName()); + if (check(address)) { + return address; } else { - return purePing(address); + return null; } } catch (Exception e) { - ActiveMQUtilLogger.LOGGER.failedToCheckAddress(e, address.toString()); - return false; + ActiveMQUtilLogger.LOGGER.failedToCheckAddress(e, straddress); + return null; + } + } + + public boolean check(InetAddress address) throws IOException, InterruptedException { + if (!hasCustomPingCommand() && isReachable(address)) { + if (logger.isTraceEnabled()) { + logger.tracef(address + " OK"); + } + return true; + } else { + return purePing(address); } } diff --git a/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/NetworkHealthTest.java b/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/NetworkHealthTest.java index b7901735b3..a1b5b41377 100644 --- a/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/NetworkHealthTest.java +++ b/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/NetworkHealthTest.java @@ -155,8 +155,7 @@ public class NetworkHealthTest { } }); check.addComponent(component); - InetAddress address = InetAddress.getByName("127.0.0.1"); - check.addAddress(address); + check.addAddress("127.0.0.1"); component.stop(); diff --git a/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/network/NetUtil.java b/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/network/NetUtil.java index 489d47b3c9..873137201a 100644 --- a/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/network/NetUtil.java +++ b/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/network/NetUtil.java @@ -109,6 +109,10 @@ public class NetUtil { public static void netUp(String ip) throws Exception { String deviceID = "lo:" + nextDevice.incrementAndGet(); + netUp(ip, deviceID); + } + + public static void netUp(String ip, String deviceID) throws Exception { if (osUsed == OS.MAC) { if (runCommand("sudo", "-n", "ifconfig", "lo0", "alias", ip) != 0) { Assert.fail("Cannot sudo ifconfig for ip " + ip); @@ -138,7 +142,9 @@ public class NetUtil { netDown(ip, device, force); } - private static void netDown(String ip, String device, boolean force) throws Exception { + public static void netDown(String ip, String device, boolean force) throws Exception { + networks.remove(ip); + if (osUsed == OS.MAC) { if (runCommand("sudo", "-n", "ifconfig", "lo0", "-alias", ip) != 0) { if (!force) { diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/management/impl/AbstractControl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/management/impl/AbstractControl.java index 3f8b83ac01..ad5b81a4f7 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/management/impl/AbstractControl.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/management/impl/AbstractControl.java @@ -70,7 +70,7 @@ public abstract class AbstractControl extends StandardMBean { protected void blockOnIO() { // the storage manager could be null on the backup on certain components - if (storageManager != null) { + if (storageManager != null && storageManager.isStarted()) { try { storageManager.waitOnOperations(); storageManager.clearContext(); diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/NetworkIsolationTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/NetworkIsolationTest.java index 95db530039..5bfd25c5b1 100644 --- a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/NetworkIsolationTest.java +++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/NetworkIsolationTest.java @@ -17,7 +17,6 @@ package org.apache.activemq.artemis.tests.integration.cluster.failover; -import java.net.InetAddress; import java.util.concurrent.TimeUnit; import org.apache.activemq.artemis.api.core.QueueConfiguration; @@ -71,7 +70,7 @@ public class NetworkIsolationTest extends FailoverTestBase { Assert.assertTrue(Wait.waitFor(liveServer::isActive)); - liveServer.getServer().getNetworkHealthCheck().addAddress(InetAddress.getByName(badAddress)); + liveServer.getServer().getNetworkHealthCheck().addAddress(badAddress); Assert.assertTrue(Wait.waitFor(() -> !liveServer.isStarted())); @@ -89,20 +88,20 @@ public class NetworkIsolationTest extends FailoverTestBase { // this block here is just to validate if ignoring loopback addresses logic is in place { - backupServer.getServer().getNetworkHealthCheck().addAddress(InetAddress.getByName("127.0.0.1")); + backupServer.getServer().getNetworkHealthCheck().addAddress("127.0.0.1"); Assert.assertTrue(AssertionLoggerHandler.findText("AMQ202001")); AssertionLoggerHandler.clear(); - backupServer.getServer().getNetworkHealthCheck().setIgnoreLoopback(true).addAddress(InetAddress.getByName("127.0.0.1")); + backupServer.getServer().getNetworkHealthCheck().setIgnoreLoopback(true).addAddress("127.0.0.1"); Assert.assertFalse(AssertionLoggerHandler.findText("AMQ202001")); backupServer.getServer().getNetworkHealthCheck().clearAddresses(); } - backupServer.getServer().getNetworkHealthCheck().addAddress(InetAddress.getByName(badAddress)); + backupServer.getServer().getNetworkHealthCheck().addAddress(badAddress); backupServer.getServer().start(); ClientSessionFactory sf = addSessionFactory(locator.createSessionFactory()); @@ -162,7 +161,7 @@ public class NetworkIsolationTest extends FailoverTestBase { Assert.assertFalse(liveServer.isStarted()); - liveServer.getServer().getNetworkHealthCheck().setIgnoreLoopback(true).addAddress(InetAddress.getByName("127.0.0.1")); + liveServer.getServer().getNetworkHealthCheck().setIgnoreLoopback(true).addAddress("127.0.0.1"); timeout = System.currentTimeMillis() + 30000; while (!liveServer.isStarted() && System.currentTimeMillis() < timeout) { diff --git a/tests/smoke-tests/pom.xml b/tests/smoke-tests/pom.xml index ee0a5a98a2..d083e8d95e 100644 --- a/tests/smoke-tests/pom.xml +++ b/tests/smoke-tests/pom.xml @@ -268,6 +268,48 @@ + + test-compile + create-dnsswitch-main-withping + + create + + + ${basedir}/target/classes/servers/dnsswitch-replicated-main-withping + true + admin + admin + true + ${basedir}/target/dnsswitch-replicated-main-withping + + --java-options + + -Djdk.net.hosts.file=${basedir}/target/dnsswitch/etc/hosts.conf -Djava.security.properties=${basedir}/target/dnsswitch/etc/zerocache.security -Djava.rmi.server.hostname=localhost + + + + + test-compile + create-dnsswitch-backup-withping + + create + + + ${basedir}/target/classes/servers/dnsswitch-replicated-backup-withping + true + admin + admin + true + ${basedir}/target/dnsswitch-replicated-backup-withping + + --java-options + + -Djdk.net.hosts.file=${basedir}/target/dnsswitch/etc/hosts.conf -Djava.security.properties=${basedir}/target/dnsswitch/etc/zerocache.security -Djava.rmi.server.hostname=localhost + + + test-compile create-dnsswitch-main-noretrydns diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/broker.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/broker.xml new file mode 100644 index 0000000000..bb08ea592c --- /dev/null +++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/broker.xml @@ -0,0 +1,139 @@ + + + + + + backup + + ./data/bindings + + ./data/journal + + ./data/largemessages + + ./data/paging + + PINGPLACE + + 1000 + + 200 + + + + + a + true + + + + + + + tcp://SECOND:61716 + tcp://FIRST:61616 + + + + + + tcp://0.0.0.0:61716 + + + admin + + password + + + + artemis + OFF + 1 + + main + + + + + + + + + + + + + + + + + + + + + + + + + + + DLQ + ExpiryQueue + 0 + + -1 + 10 + PAGE + true + true + true + true + + + + DLQ + ExpiryQueue + 0 + + 10MB + 1MB + + 10 + PAGE + true + true + true + true + + + + +
+ + +
+
+ + + +
+ + + diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/management.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/management.xml new file mode 100644 index 0000000000..14bbaf2218 --- /dev/null +++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/management.xml @@ -0,0 +1,20 @@ + + + + + \ No newline at end of file diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup/broker.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup/broker.xml index 36a07a6f6c..b24398eb75 100644 --- a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup/broker.xml +++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup/broker.xml @@ -41,14 +41,14 @@ under the License. - tcp://SECOND:61616 + tcp://SECOND:61716 tcp://FIRST:61616 - tcp://SECOND:61616 + tcp://0.0.0.0:61716 admin @@ -58,6 +58,8 @@ under the License. artemis + 100 + 500 OFF 1 diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-noretrydns/broker.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-noretrydns/broker.xml index bb5995e195..44029f6a0b 100644 --- a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-noretrydns/broker.xml +++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-noretrydns/broker.xml @@ -43,12 +43,12 @@ under the License. tcp://FIRST:61616 - tcp://SECOND:61616 + tcp://SECOND:61716 - tcp://FIRST:61616 + tcp://0.0.0.0:61616 admin diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/broker.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/broker.xml new file mode 100644 index 0000000000..174e8d3133 --- /dev/null +++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/broker.xml @@ -0,0 +1,139 @@ + + + + + + live + + ./data/bindings + + ./data/journal + + ./data/largemessages + + ./data/paging + + + + + a + true + true + + + + + PINGPLACE + + 1000 + + 200 + + + + tcp://FIRST:61616 + tcp://SECOND:61716 + + + + + tcp://0.0.0.0:61616 + + + admin + + password + + + + artemis + OFF + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + DLQ + ExpiryQueue + 0 + + -1 + 10 + PAGE + true + true + true + true + + + + DLQ + ExpiryQueue + 0 + + 10MB + 1MB + + 10 + PAGE + true + true + true + true + + + + +
+ + +
+
+ + + +
+ + + diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/management.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/management.xml new file mode 100644 index 0000000000..576f1e5995 --- /dev/null +++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/management.xml @@ -0,0 +1,20 @@ + + + + + \ No newline at end of file diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main/broker.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main/broker.xml index bb5995e195..44029f6a0b 100644 --- a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main/broker.xml +++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main/broker.xml @@ -43,12 +43,12 @@ under the License. tcp://FIRST:61616 - tcp://SECOND:61616 + tcp://SECOND:61716 - tcp://FIRST:61616 + tcp://0.0.0.0:61616 admin diff --git a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/dnsswitch/DNSSwitchTest.java b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/dnsswitch/DNSSwitchTest.java index 31a76d07d3..aa51d9bc37 100644 --- a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/dnsswitch/DNSSwitchTest.java +++ b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/dnsswitch/DNSSwitchTest.java @@ -94,12 +94,17 @@ public class DNSSwitchTest extends SmokeTestBase { private static final String SERVER_LIVE_NORETRYDNS = "dnsswitch-replicated-main-noretrydns"; private static final String SERVER_BACKUP = "dnsswitch-replicated-backup"; + private static final String SERVER_LIVE_PING = "dnsswitch-replicated-main-withping"; + private static final String SERVER_BACKUP_PING = "dnsswitch-replicated-backup-withping"; + // 192.0.2.0 is reserved for documentation (and testing on this case). private static final String FIRST_IP = "192.0.2.0"; private static final String SECOND_IP = "192.0.3.0"; private static final String THIRD_IP = "192.0.3.0"; private static final String FOURTH_IP = "192.0.4.0"; + private static final String INVALID_IP = "203.0.113.0"; + private static String serverLocation; @Rule @@ -122,7 +127,6 @@ public class DNSSwitchTest extends SmokeTestBase { serverControl.isActive(); // making one call to make sure it's working return serverControl; } catch (Throwable e) { - System.err.println("Retrying error : " + e.getMessage()); lastException = e; Thread.sleep(500); } @@ -235,24 +239,20 @@ public class DNSSwitchTest extends SmokeTestBase { cleanupData(SERVER_LIVE); cleanupData(SERVER_LIVE_NORETRYDNS); cleanupData(SERVER_BACKUP); + cleanupData(SERVER_LIVE_PING); + cleanupData(SERVER_BACKUP_PING); } @Test public void testBackupRedefinition() throws Throwable { - System.out.println(System.getProperty("java.security.properties")); - spawnRun(serverLocation, "testBackupRedefinition", getServerLocation(SERVER_LIVE), getServerLocation(SERVER_BACKUP)); } public static void testBackupRedefinition(String[] args) throws Throwable { - NetUtil.netUp(FIRST_IP); - NetUtil.netUp(SECOND_IP); - // NetUtil.netUp(THIRD_IP); + NetUtil.netUp(FIRST_IP, "lo:first"); + NetUtil.netUp(SECOND_IP, "lo:second"); saveConf(hostsFile, FIRST_IP, "FIRST", SECOND_IP, "SECOND"); - //System.out.println("Waiting here"); - //Thread.sleep(300_000); - Process serverLive = null; Process serverBackup = null; @@ -263,7 +263,7 @@ public class DNSSwitchTest extends SmokeTestBase { connectAndWaitBackup(); saveConf(hostsFile, FIRST_IP, "FIRST", THIRD_IP, "SECOND"); - NetUtil.netDown(SECOND_IP, true); + NetUtil.netDown(SECOND_IP, "lo:second", true); serverBackup.destroyForcibly(); Thread.sleep(1000); // wait some time at least until a reconnection is in place @@ -280,11 +280,6 @@ public class DNSSwitchTest extends SmokeTestBase { connectAndWaitBackup(); - //waitForTopology(connectionFactory.getServerLocator().getTopology(), 60_000, 1, 1); - - //System.out.println("I'm here!!!"); - //Thread.sleep(300_000); - ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory("tcp://FIRST:61616?ha=true"); Assert.assertTrue(connectionFactory.getServerLocator().isHA()); Connection connection = connectionFactory.createConnection(); @@ -299,7 +294,7 @@ public class DNSSwitchTest extends SmokeTestBase { producer.send(session.createTextMessage("hello")); session.commit(); - NetUtil.netUp(THIRD_IP); + NetUtil.netUp(THIRD_IP, "lo:third"); serverLive.destroyForcibly(); Wait.assertTrue(backupControl::isActive); @@ -320,7 +315,7 @@ public class DNSSwitchTest extends SmokeTestBase { errors++; Assert.assertTrue(errors < 20); // I would accept one or two errors, but the code must connect itself connection.close(); - connectionFactory = new ActiveMQConnectionFactory("tcp://SECOND:61616?ha=true"); + connectionFactory = new ActiveMQConnectionFactory("tcp://SECOND:61716?ha=true"); connection = connectionFactory.createConnection(); connection.start(); session = connection.createSession(true, Session.SESSION_TRANSACTED); @@ -342,20 +337,15 @@ public class DNSSwitchTest extends SmokeTestBase { @Test public void testBackupRedefinition2() throws Throwable { - System.out.println(System.getProperty("java.security.properties")); - spawnRun(serverLocation, "testBackupRedefinition2", getServerLocation(SERVER_LIVE), getServerLocation(SERVER_BACKUP)); } public static void testBackupRedefinition2(String[] args) throws Throwable { - NetUtil.netUp(FIRST_IP); - NetUtil.netUp(SECOND_IP); - NetUtil.netUp(THIRD_IP); + NetUtil.netUp(FIRST_IP, "lo:first"); + NetUtil.netUp(SECOND_IP, "lo:second"); + NetUtil.netUp(THIRD_IP, "lo:third"); saveConf(hostsFile, FIRST_IP, "FIRST", SECOND_IP, "SECOND"); - //System.out.println("Waiting here"); - //Thread.sleep(300_000); - Process serverLive = null; Process serverBackup = null; @@ -366,7 +356,7 @@ public class DNSSwitchTest extends SmokeTestBase { connectAndWaitBackup(); saveConf(hostsFile, FIRST_IP, "FIRST", THIRD_IP, "SECOND"); - NetUtil.netDown(SECOND_IP, true); + NetUtil.netDown(SECOND_IP, "lo:second", true); serverBackup.destroyForcibly(); Thread.sleep(1000); // wait some time at least until a reconnection is in place @@ -391,11 +381,6 @@ public class DNSSwitchTest extends SmokeTestBase { Wait.assertTrue(backupControl::isStarted); Wait.assertTrue(backupControl::isReplicaSync); - //waitForTopology(connectionFactory.getServerLocator().getTopology(), 60_000, 1, 1); - - //System.out.println("I'm here!!!"); - //Thread.sleep(300_000); - ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory("tcp://FIRST:61616?ha=true"); Assert.assertTrue(connectionFactory.getServerLocator().isHA()); Connection connection = connectionFactory.createConnection(); @@ -410,7 +395,7 @@ public class DNSSwitchTest extends SmokeTestBase { producer.send(session.createTextMessage("hello")); session.commit(); - NetUtil.netUp(THIRD_IP); + NetUtil.netUp(THIRD_IP, "lo:third"); serverLive.destroyForcibly(); Wait.assertTrue(backupControl::isActive); @@ -431,7 +416,7 @@ public class DNSSwitchTest extends SmokeTestBase { errors++; Assert.assertTrue(errors < 20); // I would accept one or two errors, but the code must connect itself connection.close(); - connectionFactory = new ActiveMQConnectionFactory("tcp://SECOND:61616?ha=true"); + connectionFactory = new ActiveMQConnectionFactory("tcp://SECOND:61716?ha=true"); connection = connectionFactory.createConnection(); connection.start(); session = connection.createSession(true, Session.SESSION_TRANSACTED); @@ -452,20 +437,15 @@ public class DNSSwitchTest extends SmokeTestBase { @Test public void testBackupRedefinition3() throws Throwable { - System.out.println(System.getProperty("java.security.properties")); - spawnRun(serverLocation, "testBackupRedefinition2", getServerLocation(SERVER_LIVE), getServerLocation(SERVER_BACKUP)); } public static void testBackupRedefinition3(String[] args) throws Throwable { - NetUtil.netUp(FIRST_IP); - NetUtil.netUp(SECOND_IP); - NetUtil.netUp(THIRD_IP); + NetUtil.netUp(FIRST_IP, "lo:first"); + NetUtil.netUp(SECOND_IP, "lo:second"); + NetUtil.netUp(THIRD_IP, "lo:third"); saveConf(hostsFile, FIRST_IP, "FIRST", SECOND_IP, "SECOND"); - //System.out.println("Waiting here"); - //Thread.sleep(300_000); - Process serverLive = null; Process serverBackup = null; @@ -476,7 +456,7 @@ public class DNSSwitchTest extends SmokeTestBase { connectAndWaitBackup(); saveConf(hostsFile, FIRST_IP, "FIRST", THIRD_IP, "SECOND"); - NetUtil.netDown(SECOND_IP, true); + NetUtil.netDown(SECOND_IP, "lo:second", true); serverBackup.destroyForcibly(); Thread.sleep(1000); // wait some time at least until a reconnection is in place @@ -501,11 +481,6 @@ public class DNSSwitchTest extends SmokeTestBase { Wait.assertTrue(backupControl::isStarted); Wait.assertTrue(backupControl::isReplicaSync); - //waitForTopology(connectionFactory.getServerLocator().getTopology(), 60_000, 1, 1); - - //System.out.println("I'm here!!!"); - //Thread.sleep(300_000); - ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory("tcp://FIRST:61616?ha=true"); Assert.assertTrue(connectionFactory.getServerLocator().isHA()); Connection connection = connectionFactory.createConnection(); @@ -520,7 +495,7 @@ public class DNSSwitchTest extends SmokeTestBase { producer.send(session.createTextMessage("hello")); session.commit(); - NetUtil.netUp(THIRD_IP); + NetUtil.netUp(THIRD_IP, "lo:third"); serverLive.destroyForcibly(); Wait.assertTrue(backupControl::isActive); @@ -564,14 +539,12 @@ public class DNSSwitchTest extends SmokeTestBase { @Test public void testCantReachBack() throws Throwable { - System.out.println(System.getProperty("java.security.properties")); - spawnRun(serverLocation, "testCantReachBack", getServerLocation(SERVER_LIVE_NORETRYDNS), getServerLocation(SERVER_BACKUP)); } public static void testCantReachBack(String[] args) throws Throwable { - NetUtil.netUp(FIRST_IP); - NetUtil.netUp(SECOND_IP); + NetUtil.netUp(FIRST_IP, "lo:first"); + NetUtil.netUp(SECOND_IP, "lo:second"); // notice there's no THIRD_IP anywhere saveConf(hostsFile, FIRST_IP, "FIRST", THIRD_IP, "SECOND"); @@ -595,6 +568,90 @@ public class DNSSwitchTest extends SmokeTestBase { connectAndWaitBackup(); + } finally { + if (serverBackup != null) { + serverBackup.destroyForcibly(); + } + if (serverLive != null) { + serverLive.destroyForcibly(); + } + } + + } + + + @Test + public void testWithPing() throws Throwable { + spawnRun(serverLocation, "testWithPing", getServerLocation(SERVER_LIVE_PING), getServerLocation(SERVER_BACKUP_PING)); + } + + public static void testWithPing(String[] args) throws Throwable { + NetUtil.netUp(FIRST_IP, "lo:first"); + NetUtil.netUp(SECOND_IP, "lo:second"); + NetUtil.netUp(THIRD_IP, "lo:third"); + + // notice there's no THIRD_IP anywhere + saveConf(hostsFile, FIRST_IP, "FIRST", SECOND_IP, "SECOND", THIRD_IP, "PINGPLACE"); + + Process serverLive = null; + Process serverBackup = null; + + try { + serverLive = ServerUtil.startServer(args[1], "live", "tcp://FIRST:61616", 30_000); + ActiveMQServerControl liveControl = getServerControl(liveURI, liveNameBuilder, 20_000); + + Wait.assertTrue(liveControl::isStarted); + + // notice the first server does not know about this server at all + serverBackup = ServerUtil.startServer(args[2], "backup", "tcp://SECOND:61716", 0); + ActiveMQServerControl backupControl = getServerControl(backupURI, backupNameBuilder, 20_000); + + Wait.assertTrue(backupControl::isStarted); + Wait.assertTrue(backupControl::isReplicaSync); + // Removing PINGPLACE from DNS + saveConf(hostsFile, FIRST_IP, "FIRST", SECOND_IP, "SECOND", INVALID_IP, "PINGPLACE"); + + Wait.assertFalse(liveControl::isStarted); + + serverBackup.destroyForcibly(); + + + //Thread.sleep(10_000); + serverLive.destroyForcibly(); + serverLive = ServerUtil.startServer(args[1], "live", "tcp://FIRST:61616", 0); + + Thread.sleep(1_000); + + + logger.debug("going to re-enable ping"); + // Enable the address just for ping now + saveConf(hostsFile, THIRD_IP, "PINGPLACE"); + liveControl = getServerControl(liveURI, liveNameBuilder, 20_000); + Wait.assertTrue(liveControl::isStarted); + + // Waiting some time as to the retry logic to kick in + Thread.sleep(5_000); + + // the backup will know about the live, but live doesn't have a direct DNS to backup.. lets see what happens + saveConf(hostsFile, FIRST_IP, "FIRST", THIRD_IP, "PINGPLACE"); + + boolean ok = false; + for (int i = 0; i < 5; i++) { + serverBackup = ServerUtil.startServer(args[2], "backup", "tcp://SECOND:61716", 0); + backupControl = getServerControl(backupURI, backupNameBuilder, 20_000); + Wait.assertTrue(backupControl::isStarted); + if (!Wait.waitFor(backupControl::isReplicaSync, 5000, 100)) { + serverBackup.destroyForcibly(); + } else { + ok = true; + break; + } + } + + Assert.assertTrue(ok); + + //connectAndWaitBackup(); + } finally { if (serverBackup != null) { serverBackup.destroyForcibly(); @@ -752,8 +809,8 @@ public class DNSSwitchTest extends SmokeTestBase { String securityProperties = System.getProperty("java.security.properties"); - if (securityProperties != null && securityProperties.equals(location + "/etc/zerocache.security3")) { - System.out.println("No need to spawn a VM, the zerocache is already in place"); + if (securityProperties != null && securityProperties.equals(location + "/etc/zerocache.security")) { + logger.info("No need to spawn a VM, the zerocache is already in place"); System.setProperty("artemis.config.location", location); USING_SPAWN = false; main(args);