From f733cac08ff03a267c95bc81437f747df38f7932 Mon Sep 17 00:00:00 2001
From: Clebert Suconic <clebertsuconic@apache.org>
Date: Wed, 26 Apr 2023 18:17:12 -0400
Subject: [PATCH] ARTEMIS-4254 Improving Transaction test with replication to
 use 3 nodes

---
 .../activemq/artemis/util/ServerUtil.java     |   6 +-
 tests/soak-tests/pom.xml                      |  69 +++++++--
 .../broker.xml                                |  26 +++-
 .../broker.xml                                |  31 ++--
 .../{standalone => backup-zero}/broker.xml    |  39 ++++-
 .../replica-tx-check/live-one/broker.xml      | 144 ++++++++++++++++++
 .../replica-tx-check/live-two/broker.xml      | 144 ++++++++++++++++++
 .../replica-tx-check/live-zero/broker.xml     | 144 ++++++++++++++++++
 .../ReplicaTXCheckTest.java                   | 109 +++++++++----
 9 files changed, 656 insertions(+), 56 deletions(-)
 rename tests/soak-tests/src/main/resources/servers/replica-tx-check/{replicated-static1 => backup-one}/broker.xml (80%)
 rename tests/soak-tests/src/main/resources/servers/replica-tx-check/{replicated-static0 => backup-two}/broker.xml (79%)
 rename tests/soak-tests/src/main/resources/servers/replica-tx-check/{standalone => backup-zero}/broker.xml (72%)
 create mode 100644 tests/soak-tests/src/main/resources/servers/replica-tx-check/live-one/broker.xml
 create mode 100644 tests/soak-tests/src/main/resources/servers/replica-tx-check/live-two/broker.xml
 create mode 100644 tests/soak-tests/src/main/resources/servers/replica-tx-check/live-zero/broker.xml
 rename tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/{replicationflow => replicaTxCheck}/ReplicaTXCheckTest.java (56%)

diff --git a/artemis-cli/src/main/java/org/apache/activemq/artemis/util/ServerUtil.java b/artemis-cli/src/main/java/org/apache/activemq/artemis/util/ServerUtil.java
index b0f45fba7c..c5575368d0 100644
--- a/artemis-cli/src/main/java/org/apache/activemq/artemis/util/ServerUtil.java
+++ b/artemis-cli/src/main/java/org/apache/activemq/artemis/util/ServerUtil.java
@@ -124,7 +124,11 @@ public class ServerUtil {
    }
 
    public static boolean waitForServerToStart(int id, String username, String password, int timeout) throws InterruptedException {
-      return waitForServerToStart("tcp://localhost:" + (61616 + id), username, password, timeout);
+      return waitForServerToStartOnPort(61616 + id, username, password, timeout);
+   }
+
+   public static boolean waitForServerToStartOnPort(int port, String username, String password, int timeout) throws InterruptedException {
+      return waitForServerToStart("tcp://localhost:" + port, username, password, timeout);
    }
 
    public static boolean waitForServerToStart(String uri, long timeout) throws InterruptedException {
diff --git a/tests/soak-tests/pom.xml b/tests/soak-tests/pom.xml
index 381133eaca..b67887cbe7 100644
--- a/tests/soak-tests/pom.xml
+++ b/tests/soak-tests/pom.xml
@@ -387,15 +387,15 @@
                </execution>
                <execution>
                   <phase>test-compile</phase>
-                  <id>created-tx-check-replica0</id>
+                  <id>create-tx-check-backup-zero</id>
                   <goals>
                      <goal>create</goal>
                   </goals>
                   <configuration>
                      <!-- this makes it easier in certain envs -->
                      <javaOptions>-Djava.net.preferIPv4Stack=true</javaOptions>
-                     <instance>${basedir}/target/replica-tx-check/replicated-static0</instance>
-                     <configuration>${basedir}/target/classes/servers/replica-tx-check/replicated-static0</configuration>
+                     <instance>${basedir}/target/replica-tx-check/backup-zero</instance>
+                     <configuration>${basedir}/target/classes/servers/replica-tx-check/backup-zero</configuration>
                      <args>
                         <arg>--java-options</arg>
                         <arg>-ea</arg>
@@ -404,15 +404,15 @@
                </execution>
                <execution>
                   <phase>test-compile</phase>
-                  <id>create-tx-check-replica1</id>
+                  <id>create-tx-check-live-zero</id>
                   <goals>
                      <goal>create</goal>
                   </goals>
                   <configuration>
                      <!-- this makes it easier in certain envs -->
                      <javaOptions>-Djava.net.preferIPv4Stack=true</javaOptions>
-                     <instance>${basedir}/target/replica-tx-check/replicated-static1</instance>
-                     <configuration>${basedir}/target/classes/servers/replica-tx-check/replicated-static1</configuration>
+                     <instance>${basedir}/target/replica-tx-check/live-zero</instance>
+                     <configuration>${basedir}/target/classes/servers/replica-tx-check/live-zero</configuration>
                      <args>
                         <arg>--java-options</arg>
                         <arg>-ea</arg>
@@ -421,15 +421,66 @@
                </execution>
                <execution>
                   <phase>test-compile</phase>
-                  <id>create-tx-check-standalone</id>
+                  <id>create-tx-check-backup-one</id>
                   <goals>
                      <goal>create</goal>
                   </goals>
                   <configuration>
                      <!-- this makes it easier in certain envs -->
                      <javaOptions>-Djava.net.preferIPv4Stack=true</javaOptions>
-                     <instance>${basedir}/target/replica-tx-check/standalone</instance>
-                     <configuration>${basedir}/target/classes/servers/replica-tx-check/standalone</configuration>
+                     <instance>${basedir}/target/replica-tx-check/backup-one</instance>
+                     <configuration>${basedir}/target/classes/servers/replica-tx-check/backup-one</configuration>
+                     <args>
+                        <arg>--java-options</arg>
+                        <arg>-ea</arg>
+                     </args>
+                  </configuration>
+               </execution>
+               <execution>
+                  <phase>test-compile</phase>
+                  <id>create-tx-check-live-one</id>
+                  <goals>
+                     <goal>create</goal>
+                  </goals>
+                  <configuration>
+                     <!-- this makes it easier in certain envs -->
+                     <javaOptions>-Djava.net.preferIPv4Stack=true</javaOptions>
+                     <instance>${basedir}/target/replica-tx-check/live-one</instance>
+                     <configuration>${basedir}/target/classes/servers/replica-tx-check/live-one</configuration>
+                     <args>
+                        <arg>--java-options</arg>
+                        <arg>-ea</arg>
+                     </args>
+                  </configuration>
+               </execution>
+               <execution>
+                  <phase>test-compile</phase>
+                  <id>create-tx-check-backup-two</id>
+                  <goals>
+                     <goal>create</goal>
+                  </goals>
+                  <configuration>
+                     <!-- this makes it easier in certain envs -->
+                     <javaOptions>-Djava.net.preferIPv4Stack=true</javaOptions>
+                     <instance>${basedir}/target/replica-tx-check/backup-two</instance>
+                     <configuration>${basedir}/target/classes/servers/replica-tx-check/backup-two</configuration>
+                     <args>
+                        <arg>--java-options</arg>
+                        <arg>-ea</arg>
+                     </args>
+                  </configuration>
+               </execution>
+               <execution>
+                  <phase>test-compile</phase>
+                  <id>create-tx-check-live-two</id>
+                  <goals>
+                     <goal>create</goal>
+                  </goals>
+                  <configuration>
+                     <!-- this makes it easier in certain envs -->
+                     <javaOptions>-Djava.net.preferIPv4Stack=true</javaOptions>
+                     <instance>${basedir}/target/replica-tx-check/live-two</instance>
+                     <configuration>${basedir}/target/classes/servers/replica-tx-check/live-two</configuration>
                      <args>
                         <arg>--java-options</arg>
                         <arg>-ea</arg>
diff --git a/tests/soak-tests/src/main/resources/servers/replica-tx-check/replicated-static1/broker.xml b/tests/soak-tests/src/main/resources/servers/replica-tx-check/backup-one/broker.xml
similarity index 80%
rename from tests/soak-tests/src/main/resources/servers/replica-tx-check/replicated-static1/broker.xml
rename to tests/soak-tests/src/main/resources/servers/replica-tx-check/backup-one/broker.xml
index 489b1d44b9..1e7058c7e1 100644
--- a/tests/soak-tests/src/main/resources/servers/replica-tx-check/replicated-static1/broker.xml
+++ b/tests/soak-tests/src/main/resources/servers/replica-tx-check/backup-one/broker.xml
@@ -37,29 +37,39 @@ under the License.
       <ha-policy>
          <replication>
             <slave>
-               <allow-failback>true</allow-failback>
-               <!-- not needed but tells the backup not to restart after failback as there will be > 0 backups saved -->
-               <max-saved-replicated-journals-size>0</max-saved-replicated-journals-size>
+               <group-name>test-pair-one</group-name>
+               <allow-failback>false</allow-failback>
+               <quorum-size>2</quorum-size>
             </slave>
          </replication>
       </ha-policy>
 
       <!-- Connectors -->
       <connectors>
-         <connector name="netty-live-connector">tcp://localhost:61616</connector>
-         <connector name="netty-connector">tcp://localhost:61617</connector>
+         <connector name="zero-live-connector">tcp://localhost:61000</connector>
+         <connector name="zero-backup-connector">tcp://localhost:61100</connector>
+         <connector name="one-live-connector">tcp://localhost:61001</connector>
+         <connector name="one-backup-connector">tcp://localhost:61101</connector>
+         <connector name="two-live-connector">tcp://localhost:61002</connector>
+         <connector name="two-backup-connector">tcp://localhost:61102</connector>
       </connectors>
 
       <!-- Acceptors -->
       <acceptors>
-         <acceptor name="netty-acceptor">tcp://localhost:61617</acceptor>
+         <acceptor name="netty-acceptor">tcp://localhost:61101</acceptor>
       </acceptors>
 
       <cluster-connections>
          <cluster-connection name="my-cluster">
-            <connector-ref>netty-connector</connector-ref>
+            <connector-ref>one-backup-connector</connector-ref>
+            <message-load-balancing>OFF</message-load-balancing>
+            <max-hops>1</max-hops>
             <static-connectors>
-               <connector-ref>netty-live-connector</connector-ref>
+               <connector-ref>zero-live-connector</connector-ref>
+               <connector-ref>zero-backup-connector</connector-ref>
+               <connector-ref>one-live-connector</connector-ref>
+               <connector-ref>two-live-connector</connector-ref>
+               <connector-ref>two-backup-connector</connector-ref>
             </static-connectors>
          </cluster-connection>
       </cluster-connections>
diff --git a/tests/soak-tests/src/main/resources/servers/replica-tx-check/replicated-static0/broker.xml b/tests/soak-tests/src/main/resources/servers/replica-tx-check/backup-two/broker.xml
similarity index 79%
rename from tests/soak-tests/src/main/resources/servers/replica-tx-check/replicated-static0/broker.xml
rename to tests/soak-tests/src/main/resources/servers/replica-tx-check/backup-two/broker.xml
index 7c6030392e..a613a2ff58 100644
--- a/tests/soak-tests/src/main/resources/servers/replica-tx-check/replicated-static0/broker.xml
+++ b/tests/soak-tests/src/main/resources/servers/replica-tx-check/backup-two/broker.xml
@@ -36,28 +36,40 @@ under the License.
 
       <ha-policy>
          <replication>
-            <master>
-               <!--we need this for auto failback-->
-               <check-for-live-server>true</check-for-live-server>
-            </master>
+            <slave>
+               <group-name>test-pair-two</group-name>
+               <allow-failback>false</allow-failback>
+               <quorum-size>2</quorum-size>
+            </slave>
          </replication>
       </ha-policy>
 
+      <!-- Connectors -->
       <connectors>
-         <connector name="netty-connector">tcp://localhost:61616</connector>
-         <connector name="netty-backup-connector">tcp://localhost:61617</connector>
+         <connector name="zero-live-connector">tcp://localhost:61000</connector>
+         <connector name="zero-backup-connector">tcp://localhost:61100</connector>
+         <connector name="one-live-connector">tcp://localhost:61001</connector>
+         <connector name="one-backup-connector">tcp://localhost:61101</connector>
+         <connector name="two-live-connector">tcp://localhost:61002</connector>
+         <connector name="two-backup-connector">tcp://localhost:61102</connector>
       </connectors>
 
       <!-- Acceptors -->
       <acceptors>
-         <acceptor name="netty-acceptor">tcp://localhost:61616</acceptor>
+         <acceptor name="netty-acceptor">tcp://localhost:61102</acceptor>
       </acceptors>
 
       <cluster-connections>
          <cluster-connection name="my-cluster">
-            <connector-ref>netty-connector</connector-ref>
+            <connector-ref>two-backup-connector</connector-ref>
+            <message-load-balancing>OFF</message-load-balancing>
+            <max-hops>1</max-hops>
             <static-connectors>
-               <connector-ref>netty-backup-connector</connector-ref>
+               <connector-ref>zero-live-connector</connector-ref>
+               <connector-ref>zero-backup-connector</connector-ref>
+               <connector-ref>one-live-connector</connector-ref>
+               <connector-ref>one-backup-connector</connector-ref>
+               <connector-ref>two-live-connector</connector-ref>
             </static-connectors>
          </cluster-connection>
       </cluster-connections>
@@ -100,7 +112,6 @@ under the License.
             <!-- with -1 only the global-max-size is in use for limiting -->
             <max-size-bytes>10MB</max-size-bytes>
             <page-size-bytes>1MB</page-size-bytes>
-
             <max-read-page-messages>-1</max-read-page-messages>
             <max-read-page-bytes>-1</max-read-page-bytes>
 
diff --git a/tests/soak-tests/src/main/resources/servers/replica-tx-check/standalone/broker.xml b/tests/soak-tests/src/main/resources/servers/replica-tx-check/backup-zero/broker.xml
similarity index 72%
rename from tests/soak-tests/src/main/resources/servers/replica-tx-check/standalone/broker.xml
rename to tests/soak-tests/src/main/resources/servers/replica-tx-check/backup-zero/broker.xml
index bd328b94d9..004d680c63 100644
--- a/tests/soak-tests/src/main/resources/servers/replica-tx-check/standalone/broker.xml
+++ b/tests/soak-tests/src/main/resources/servers/replica-tx-check/backup-zero/broker.xml
@@ -30,16 +30,51 @@ under the License.
 
       <paging-directory>./data/paging</paging-directory>
 
+      <cluster-user>exampleUser</cluster-user>
+
+      <cluster-password>secret</cluster-password>
+
+      <ha-policy>
+         <replication>
+            <slave>
+               <group-name>test-pair-zero</group-name>
+               <allow-failback>false</allow-failback>
+               <quorum-size>2</quorum-size>
+            </slave>
+         </replication>
+      </ha-policy>
+
       <!-- Connectors -->
       <connectors>
-         <connector name="netty-connector">tcp://localhost:61615</connector>
+         <connector name="zero-live-connector">tcp://localhost:61000</connector>
+         <connector name="zero-backup-connector">tcp://localhost:61100</connector>
+         <connector name="one-live-connector">tcp://localhost:61001</connector>
+         <connector name="one-backup-connector">tcp://localhost:61101</connector>
+         <connector name="two-live-connector">tcp://localhost:61002</connector>
+         <connector name="two-backup-connector">tcp://localhost:61102</connector>
       </connectors>
 
       <!-- Acceptors -->
       <acceptors>
-         <acceptor name="netty-acceptor">tcp://localhost:61615</acceptor>
+         <acceptor name="netty-acceptor">tcp://localhost:61100</acceptor>
       </acceptors>
 
+      <cluster-connections>
+         <cluster-connection name="my-cluster">
+            <connector-ref>zero-backup-connector</connector-ref>
+            <message-load-balancing>OFF</message-load-balancing>
+            <max-hops>1</max-hops>
+            <static-connectors>
+               <connector-ref>zero-live-connector</connector-ref>
+               <connector-ref>one-live-connector</connector-ref>
+               <connector-ref>one-backup-connector</connector-ref>
+               <connector-ref>two-live-connector</connector-ref>
+               <connector-ref>two-backup-connector</connector-ref>
+            </static-connectors>
+         </cluster-connection>
+      </cluster-connections>
+      <!-- Other config -->
+
       <security-settings>
          <!--security for example queue-->
          <security-setting match="#">
diff --git a/tests/soak-tests/src/main/resources/servers/replica-tx-check/live-one/broker.xml b/tests/soak-tests/src/main/resources/servers/replica-tx-check/live-one/broker.xml
new file mode 100644
index 0000000000..e5155fdb5d
--- /dev/null
+++ b/tests/soak-tests/src/main/resources/servers/replica-tx-check/live-one/broker.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+--><configuration xmlns="urn:activemq" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:activemq /schema/artemis-server.xsd">
+
+
+
+   <core xmlns="urn:activemq:core">
+
+      <bindings-directory>./data/bindings</bindings-directory>
+
+      <journal-directory>./data/journal</journal-directory>
+
+      <large-messages-directory>./data/largemessages</large-messages-directory>
+
+      <paging-directory>./data/paging</paging-directory>
+
+      <cluster-user>exampleUser</cluster-user>
+
+      <cluster-password>secret</cluster-password>
+
+      <ha-policy>
+         <replication>
+            <master>
+               <group-name>test-pair-one</group-name>
+               <check-for-live-server>false</check-for-live-server>
+               <vote-on-replication-failure>false</vote-on-replication-failure>
+               <quorum-size>2</quorum-size>
+            </master>
+         </replication>
+      </ha-policy>
+
+      <!-- Connectors -->
+      <connectors>
+         <connector name="zero-live-connector">tcp://localhost:61000</connector>
+         <connector name="zero-backup-connector">tcp://localhost:61100</connector>
+         <connector name="one-live-connector">tcp://localhost:61001</connector>
+         <connector name="one-backup-connector">tcp://localhost:61101</connector>
+         <connector name="two-live-connector">tcp://localhost:61002</connector>
+         <connector name="two-backup-connector">tcp://localhost:61102</connector>
+      </connectors>
+
+      <!-- Acceptors -->
+      <acceptors>
+         <acceptor name="netty-acceptor">tcp://localhost:61001</acceptor>
+      </acceptors>
+
+      <cluster-connections>
+         <cluster-connection name="my-cluster">
+            <connector-ref>one-live-connector</connector-ref>
+            <message-load-balancing>OFF</message-load-balancing>
+            <max-hops>1</max-hops>
+            <static-connectors>
+               <connector-ref>zero-live-connector</connector-ref>
+               <connector-ref>zero-backup-connector</connector-ref>
+               <connector-ref>one-backup-connector</connector-ref>
+               <connector-ref>two-live-connector</connector-ref>
+               <connector-ref>two-backup-connector</connector-ref>
+            </static-connectors>
+         </cluster-connection>
+      </cluster-connections>
+      <!-- Other config -->
+
+      <security-settings>
+         <!--security for example queue-->
+         <security-setting match="#">
+            <permission type="createNonDurableQueue" roles="amq, guest"/>
+            <permission type="deleteNonDurableQueue" roles="amq, guest"/>
+            <permission type="createDurableQueue" roles="amq, guest"/>
+            <permission type="deleteDurableQueue" roles="amq, guest"/>
+            <permission type="createAddress" roles="amq, guest"/>
+            <permission type="deleteAddress" roles="amq, guest"/>
+            <permission type="consume" roles="amq, guest"/>
+            <permission type="browse" roles="amq, guest"/>
+            <permission type="send" roles="amq, guest"/>
+            <!-- we need this otherwise ./artemis data imp wouldn't work -->
+            <permission type="manage" roles="amq"/>
+         </security-setting>
+      </security-settings>
+      <address-settings>
+         <!-- if you define auto-create on certain queues, management has to be auto-create -->
+         <address-setting match="activemq.management#">
+            <dead-letter-address>DLQ</dead-letter-address>
+            <expiry-address>ExpiryQueue</expiry-address>
+            <redelivery-delay>0</redelivery-delay>
+            <!-- with -1 only the global-max-size is in use for limiting -->
+            <max-size-bytes>-1</max-size-bytes>
+            <message-counter-history-day-limit>10</message-counter-history-day-limit>
+            <address-full-policy>PAGE</address-full-policy>
+            <auto-create-queues>true</auto-create-queues>
+            <auto-create-addresses>true</auto-create-addresses>
+         </address-setting>
+         <!--default for catch all-->
+         <address-setting match="#">
+            <dead-letter-address>DLQ</dead-letter-address>
+            <expiry-address>ExpiryQueue</expiry-address>
+            <redelivery-delay>0</redelivery-delay>
+            <!-- with -1 only the global-max-size is in use for limiting -->
+            <max-size-bytes>10MB</max-size-bytes>
+            <page-size-bytes>1MB</page-size-bytes>
+
+            <max-read-page-messages>-1</max-read-page-messages>
+            <max-read-page-bytes>-1</max-read-page-bytes>
+
+            <message-counter-history-day-limit>10</message-counter-history-day-limit>
+            <address-full-policy>PAGE</address-full-policy>
+            <auto-create-queues>true</auto-create-queues>
+            <auto-create-addresses>true</auto-create-addresses>
+         </address-setting>
+      </address-settings>
+
+      <addresses>
+         <address name="exampleTopic">
+            <multicast>
+            </multicast>
+         </address>
+         <address name="DLQ">
+            <anycast>
+               <queue name="DLQ"/>
+            </anycast>
+         </address>
+         <address name="exampleQueue">
+            <anycast>
+               <queue name="exampleQueue"/>
+            </anycast>
+         </address>
+      </addresses>
+   </core>
+</configuration>
diff --git a/tests/soak-tests/src/main/resources/servers/replica-tx-check/live-two/broker.xml b/tests/soak-tests/src/main/resources/servers/replica-tx-check/live-two/broker.xml
new file mode 100644
index 0000000000..2cae594ca5
--- /dev/null
+++ b/tests/soak-tests/src/main/resources/servers/replica-tx-check/live-two/broker.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+--><configuration xmlns="urn:activemq" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:activemq /schema/artemis-server.xsd">
+
+
+
+   <core xmlns="urn:activemq:core">
+
+      <bindings-directory>./data/bindings</bindings-directory>
+
+      <journal-directory>./data/journal</journal-directory>
+
+      <large-messages-directory>./data/largemessages</large-messages-directory>
+
+      <paging-directory>./data/paging</paging-directory>
+
+      <cluster-user>exampleUser</cluster-user>
+
+      <cluster-password>secret</cluster-password>
+
+      <ha-policy>
+         <replication>
+            <master>
+               <group-name>test-pair-one</group-name>
+               <check-for-live-server>false</check-for-live-server>
+               <vote-on-replication-failure>false</vote-on-replication-failure>
+               <quorum-size>2</quorum-size>
+            </master>
+         </replication>
+      </ha-policy>
+
+      <!-- Connectors -->
+      <connectors>
+         <connector name="zero-live-connector">tcp://localhost:61000</connector>
+         <connector name="zero-backup-connector">tcp://localhost:61100</connector>
+         <connector name="one-live-connector">tcp://localhost:61001</connector>
+         <connector name="one-backup-connector">tcp://localhost:61101</connector>
+         <connector name="two-live-connector">tcp://localhost:61002</connector>
+         <connector name="two-backup-connector">tcp://localhost:61102</connector>
+      </connectors>
+
+      <!-- Acceptors -->
+      <acceptors>
+         <acceptor name="netty-acceptor">tcp://localhost:61002</acceptor>
+      </acceptors>
+
+      <cluster-connections>
+         <cluster-connection name="my-cluster">
+            <connector-ref>two-live-connector</connector-ref>
+            <message-load-balancing>OFF</message-load-balancing>
+            <max-hops>1</max-hops>
+            <static-connectors>
+               <connector-ref>zero-live-connector</connector-ref>
+               <connector-ref>zero-backup-connector</connector-ref>
+               <connector-ref>one-backup-connector</connector-ref>
+               <connector-ref>one-live-connector</connector-ref>
+               <connector-ref>two-backup-connector</connector-ref>
+            </static-connectors>
+         </cluster-connection>
+      </cluster-connections>
+      <!-- Other config -->
+
+      <security-settings>
+         <!--security for example queue-->
+         <security-setting match="#">
+            <permission type="createNonDurableQueue" roles="amq, guest"/>
+            <permission type="deleteNonDurableQueue" roles="amq, guest"/>
+            <permission type="createDurableQueue" roles="amq, guest"/>
+            <permission type="deleteDurableQueue" roles="amq, guest"/>
+            <permission type="createAddress" roles="amq, guest"/>
+            <permission type="deleteAddress" roles="amq, guest"/>
+            <permission type="consume" roles="amq, guest"/>
+            <permission type="browse" roles="amq, guest"/>
+            <permission type="send" roles="amq, guest"/>
+            <!-- we need this otherwise ./artemis data imp wouldn't work -->
+            <permission type="manage" roles="amq"/>
+         </security-setting>
+      </security-settings>
+      <address-settings>
+         <!-- if you define auto-create on certain queues, management has to be auto-create -->
+         <address-setting match="activemq.management#">
+            <dead-letter-address>DLQ</dead-letter-address>
+            <expiry-address>ExpiryQueue</expiry-address>
+            <redelivery-delay>0</redelivery-delay>
+            <!-- with -1 only the global-max-size is in use for limiting -->
+            <max-size-bytes>-1</max-size-bytes>
+            <message-counter-history-day-limit>10</message-counter-history-day-limit>
+            <address-full-policy>PAGE</address-full-policy>
+            <auto-create-queues>true</auto-create-queues>
+            <auto-create-addresses>true</auto-create-addresses>
+         </address-setting>
+         <!--default for catch all-->
+         <address-setting match="#">
+            <dead-letter-address>DLQ</dead-letter-address>
+            <expiry-address>ExpiryQueue</expiry-address>
+            <redelivery-delay>0</redelivery-delay>
+            <!-- with -1 only the global-max-size is in use for limiting -->
+            <max-size-bytes>10MB</max-size-bytes>
+            <page-size-bytes>1MB</page-size-bytes>
+
+            <max-read-page-messages>-1</max-read-page-messages>
+            <max-read-page-bytes>-1</max-read-page-bytes>
+
+            <message-counter-history-day-limit>10</message-counter-history-day-limit>
+            <address-full-policy>PAGE</address-full-policy>
+            <auto-create-queues>true</auto-create-queues>
+            <auto-create-addresses>true</auto-create-addresses>
+         </address-setting>
+      </address-settings>
+
+      <addresses>
+         <address name="exampleTopic">
+            <multicast>
+            </multicast>
+         </address>
+         <address name="DLQ">
+            <anycast>
+               <queue name="DLQ"/>
+            </anycast>
+         </address>
+         <address name="exampleQueue">
+            <anycast>
+               <queue name="exampleQueue"/>
+            </anycast>
+         </address>
+      </addresses>
+   </core>
+</configuration>
diff --git a/tests/soak-tests/src/main/resources/servers/replica-tx-check/live-zero/broker.xml b/tests/soak-tests/src/main/resources/servers/replica-tx-check/live-zero/broker.xml
new file mode 100644
index 0000000000..03d2973246
--- /dev/null
+++ b/tests/soak-tests/src/main/resources/servers/replica-tx-check/live-zero/broker.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+--><configuration xmlns="urn:activemq" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:activemq /schema/artemis-server.xsd">
+
+
+
+   <core xmlns="urn:activemq:core">
+
+      <bindings-directory>./data/bindings</bindings-directory>
+
+      <journal-directory>./data/journal</journal-directory>
+
+      <large-messages-directory>./data/largemessages</large-messages-directory>
+
+      <paging-directory>./data/paging</paging-directory>
+
+      <cluster-user>exampleUser</cluster-user>
+
+      <cluster-password>secret</cluster-password>
+
+      <ha-policy>
+         <replication>
+            <master>
+               <group-name>test-pair-zero</group-name>
+               <check-for-live-server>false</check-for-live-server>
+               <vote-on-replication-failure>false</vote-on-replication-failure>
+               <quorum-size>2</quorum-size>
+            </master>
+         </replication>
+      </ha-policy>
+
+      <!-- Connectors -->
+      <connectors>
+         <connector name="zero-live-connector">tcp://localhost:61000</connector>
+         <connector name="zero-backup-connector">tcp://localhost:61100</connector>
+         <connector name="one-live-connector">tcp://localhost:61001</connector>
+         <connector name="one-backup-connector">tcp://localhost:61101</connector>
+         <connector name="two-live-connector">tcp://localhost:61002</connector>
+         <connector name="two-backup-connector">tcp://localhost:61102</connector>
+      </connectors>
+
+      <!-- Acceptors -->
+      <acceptors>
+         <acceptor name="netty-acceptor">tcp://localhost:61000</acceptor>
+      </acceptors>
+
+      <cluster-connections>
+         <cluster-connection name="my-cluster">
+            <connector-ref>zero-live-connector</connector-ref>
+            <message-load-balancing>OFF</message-load-balancing>
+            <max-hops>1</max-hops>
+            <static-connectors>
+               <connector-ref>zero-live-connector</connector-ref>
+               <connector-ref>one-live-connector</connector-ref>
+               <connector-ref>one-backup-connector</connector-ref>
+               <connector-ref>two-live-connector</connector-ref>
+               <connector-ref>two-backup-connector</connector-ref>
+            </static-connectors>
+         </cluster-connection>
+      </cluster-connections>
+      <!-- Other config -->
+
+      <security-settings>
+         <!--security for example queue-->
+         <security-setting match="#">
+            <permission type="createNonDurableQueue" roles="amq, guest"/>
+            <permission type="deleteNonDurableQueue" roles="amq, guest"/>
+            <permission type="createDurableQueue" roles="amq, guest"/>
+            <permission type="deleteDurableQueue" roles="amq, guest"/>
+            <permission type="createAddress" roles="amq, guest"/>
+            <permission type="deleteAddress" roles="amq, guest"/>
+            <permission type="consume" roles="amq, guest"/>
+            <permission type="browse" roles="amq, guest"/>
+            <permission type="send" roles="amq, guest"/>
+            <!-- we need this otherwise ./artemis data imp wouldn't work -->
+            <permission type="manage" roles="amq"/>
+         </security-setting>
+      </security-settings>
+      <address-settings>
+         <!-- if you define auto-create on certain queues, management has to be auto-create -->
+         <address-setting match="activemq.management#">
+            <dead-letter-address>DLQ</dead-letter-address>
+            <expiry-address>ExpiryQueue</expiry-address>
+            <redelivery-delay>0</redelivery-delay>
+            <!-- with -1 only the global-max-size is in use for limiting -->
+            <max-size-bytes>-1</max-size-bytes>
+            <message-counter-history-day-limit>10</message-counter-history-day-limit>
+            <address-full-policy>PAGE</address-full-policy>
+            <auto-create-queues>true</auto-create-queues>
+            <auto-create-addresses>true</auto-create-addresses>
+         </address-setting>
+         <!--default for catch all-->
+         <address-setting match="#">
+            <dead-letter-address>DLQ</dead-letter-address>
+            <expiry-address>ExpiryQueue</expiry-address>
+            <redelivery-delay>0</redelivery-delay>
+            <!-- with -1 only the global-max-size is in use for limiting -->
+            <max-size-bytes>10MB</max-size-bytes>
+            <page-size-bytes>1MB</page-size-bytes>
+
+            <max-read-page-messages>-1</max-read-page-messages>
+            <max-read-page-bytes>-1</max-read-page-bytes>
+
+            <message-counter-history-day-limit>10</message-counter-history-day-limit>
+            <address-full-policy>PAGE</address-full-policy>
+            <auto-create-queues>true</auto-create-queues>
+            <auto-create-addresses>true</auto-create-addresses>
+         </address-setting>
+      </address-settings>
+
+      <addresses>
+         <address name="exampleTopic">
+            <multicast>
+            </multicast>
+         </address>
+         <address name="DLQ">
+            <anycast>
+               <queue name="DLQ"/>
+            </anycast>
+         </address>
+         <address name="exampleQueue">
+            <anycast>
+               <queue name="exampleQueue"/>
+            </anycast>
+         </address>
+      </addresses>
+   </core>
+</configuration>
diff --git a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/replicationflow/ReplicaTXCheckTest.java b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/replicaTxCheck/ReplicaTXCheckTest.java
similarity index 56%
rename from tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/replicationflow/ReplicaTXCheckTest.java
rename to tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/replicaTxCheck/ReplicaTXCheckTest.java
index 818ad24e59..5c87ae9ade 100644
--- a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/replicationflow/ReplicaTXCheckTest.java
+++ b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/replicaTxCheck/ReplicaTXCheckTest.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.activemq.artemis.tests.soak.replicationflow;
+package org.apache.activemq.artemis.tests.soak.replicaTxCheck;
 
 import javax.jms.Connection;
 import javax.jms.ConnectionFactory;
@@ -26,11 +26,11 @@ import javax.jms.Session;
 import javax.jms.TextMessage;
 import javax.jms.Topic;
 import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.concurrent.TimeUnit;
+import java.util.HashSet;
 
 import org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory;
 import org.apache.activemq.artemis.tests.soak.SoakTestBase;
+import org.apache.activemq.artemis.util.ServerUtil;
 import org.apache.qpid.jms.JmsConnectionFactory;
 import org.junit.After;
 import org.junit.Assert;
@@ -43,16 +43,21 @@ public class ReplicaTXCheckTest  extends SoakTestBase {
 
    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
-   public static final String SERVER_NAME_0 = "replica-tx-check/replicated-static0";
-   public static final String SERVER_NAME_1 = "replica-tx-check/replicated-static1";
-   public static final String SERVER_NAME_2 = "replica-tx-check/standalone";
+   public static final String SERVER_NAME_0 = "replica-tx-check/backup-zero";
+   public static final String SERVER_NAME_1 = "replica-tx-check/live-zero";
+   public static final String SERVER_NAME_2 = "replica-tx-check/backup-one";
+   public static final String SERVER_NAME_3 = "replica-tx-check/live-one";
+   public static final String SERVER_NAME_4 = "replica-tx-check/backup-two";
+   public static final String SERVER_NAME_5 = "replica-tx-check/live-two";
 
-   ArrayList<ReplicationFlowControlTest.Consumer> consumers = new ArrayList<>();
    private static Process server0;
    private static Process server1;
    private static Process server2;
+   private static Process server3;
+   private static Process server4;
+   private static Process server5;
 
-   int NUMBER_OF_MESSAGES = 300;
+   int NUMBER_OF_MESSAGES = 1000;
    int KILL_AT = 100;
 
    @Before
@@ -60,11 +65,22 @@ public class ReplicaTXCheckTest  extends SoakTestBase {
       cleanupData(SERVER_NAME_0);
       cleanupData(SERVER_NAME_1);
       cleanupData(SERVER_NAME_2);
+      cleanupData(SERVER_NAME_3);
+      cleanupData(SERVER_NAME_4);
+      cleanupData(SERVER_NAME_5);
       disableCheckThread();
 
-      server0 = startServer(SERVER_NAME_0, 0, 30000);
-      server1 = startServer(SERVER_NAME_1, 1, 0);
-      server2 = startServer(SERVER_NAME_2, -1, 30000);
+      server0 = startServer(SERVER_NAME_0, 0, 0);
+      server1 = startServer(SERVER_NAME_1, 0, 0);
+      Assert.assertTrue(ServerUtil.waitForServerToStartOnPort(61000, null, null, 15000));
+
+      server2 = startServer(SERVER_NAME_2, 0, 0);
+      server3 = startServer(SERVER_NAME_3, 0, 0);
+      Assert.assertTrue(ServerUtil.waitForServerToStartOnPort(61001, null, null, 15000));
+
+      server4 = startServer(SERVER_NAME_4, 0, 0);
+      server4 = startServer(SERVER_NAME_5, 0, 0);
+      Assert.assertTrue(ServerUtil.waitForServerToStartOnPort(61002, null, null, 15000));
    }
 
    @After
@@ -75,43 +91,75 @@ public class ReplicaTXCheckTest  extends SoakTestBase {
 
    @Test
    public void testTXCheckAMQP() throws Exception {
-      testTXCheck("AMQP");
+      testTXCheck("AMQP", true, true);
    }
 
    @Test
    public void testTXCheckCORE() throws Exception {
-      testTXCheck("CORE");
+      testTXCheck("CORE", true, true);
    }
 
-   void testTXCheck(String protocol) throws Exception {
+   // a second variation of the test will invert the servers used and use a hard kill (halt) instead of stop
+   @Test
+   public void testTXCheckAMQP_2() throws Exception {
+      testTXCheck("AMQP", false, false);
+   }
 
-      ConnectionFactory replicaPairCF;
-      ConnectionFactory standaloneSource;
+   // a second variation of the test will invert the servers used and use a hard kill (halt) instead of stop
+   @Test
+   public void testTXCheckCORE_2() throws Exception {
+      testTXCheck("CORE", false, false);
+   }
+
+   /**
+    * this test is using three pairs of servers.
+    * It will send messages to one pair, then it consumes from that pair and sends to a second pair
+    * if killTarget==true the target pair is the one that's being killed, otherwise is the one with the consumers
+    * if useStop==true then the server is stopped with a regular stop call, otherwise it's halted
+    */
+   void testTXCheck(String protocol, boolean killTarget, boolean useStop) throws Exception {
+
+      ConnectionFactory pair0;
+      ConnectionFactory pair1;
 
       switch(protocol) {
          case "AMQP":
-            replicaPairCF = new JmsConnectionFactory("failover:(amqp://localhost:61616,amqp://localhost:61617)");
-            standaloneSource = new JmsConnectionFactory("amqp://localhost:61615");
+            pair0 = new JmsConnectionFactory("failover:(amqp://localhost:61000,amqp://localhost:61100)");
+            pair1 = new JmsConnectionFactory("amqp://localhost:61001");
             break;
          case "CORE":
          default:
-            replicaPairCF = new ActiveMQConnectionFactory("tcp://localhost:61616?ha=true&reconnectAttempts=-1");
-            standaloneSource = new ActiveMQConnectionFactory("tcp://localhost:61615");
+            pair0 = new ActiveMQConnectionFactory("tcp://localhost:61000?ha=true&reconnectAttempts=-1");
+            pair1 = new ActiveMQConnectionFactory("tcp://localhost:61001");
       }
 
-      try (Connection sourceConnetion = standaloneSource.createConnection()) {
+      ConnectionFactory sourceCF;
+      ConnectionFactory targetCF;
+
+
+      if (killTarget) {
+         sourceCF = pair1;
+         targetCF = pair0;
+      } else {
+         sourceCF = pair0;
+         targetCF = pair1;
+      }
+
+      try (Connection sourceConnetion = sourceCF.createConnection()) {
          Session session = sourceConnetion.createSession(true, Session.SESSION_TRANSACTED);
          Queue queue = session.createQueue("exampleQueue");
          MessageProducer producer = session.createProducer(queue);
          for (int i = 0; i < NUMBER_OF_MESSAGES; i++) {
-            producer.send(session.createTextMessage("hello " + i));
+            TextMessage message = session.createTextMessage("hello " + i);
+            message.setIntProperty("i", i);
+            producer.send(message);
          }
          session.commit();
       }
 
-      try (Connection sourceConnection = standaloneSource.createConnection();
+      try (Connection sourceConnection = sourceCF.createConnection();
            Session sourceSession = sourceConnection.createSession(true, Session.SESSION_TRANSACTED);
-           Connection targetConnection = replicaPairCF.createConnection();
+           Connection targetConnection = targetCF.createConnection();
            Session targetSession = targetConnection.createSession(true, Session.SESSION_TRANSACTED)) {
 
          sourceConnection.start();
@@ -139,8 +187,11 @@ public class ReplicaTXCheckTest  extends SoakTestBase {
                   sourceSession.commit();
                }
                if (i == KILL_AT) {
-                  server0.destroyForcibly();
-                  server0.waitFor(10, TimeUnit.SECONDS);
+                  if (useStop) {
+                     stopServerWithFile(getServerLocation(SERVER_NAME_1));
+                  } else {
+                     server1.destroyForcibly();
+                  }
                }
                if (message.getText().equals("hello " + (NUMBER_OF_MESSAGES - 1))) {
                   logger.info("got to the end");
@@ -153,6 +204,7 @@ public class ReplicaTXCheckTest  extends SoakTestBase {
             }
          }
 
+         HashSet<Integer> received = new HashSet<>();
          int rec = 0;
          while (true) {
             TextMessage message = (TextMessage) subscription.receive(100);
@@ -160,9 +212,14 @@ public class ReplicaTXCheckTest  extends SoakTestBase {
                logger.info("Received {} messages", rec);
                break;
             }
+            received.add(message.getIntProperty("i"));
             rec++;
          }
          targetSession.commit();
+
+         for (i = 0; i < NUMBER_OF_MESSAGES; i++) {
+            Assert.assertTrue(received.contains(i));
+         }
          // we could receive duplicates, but not lose messages
          Assert.assertTrue(rec >= NUMBER_OF_MESSAGES);
       }