Apache
/
activemq-artemis
mirror of https://github.com/apache/activemq-artemis.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ARTEMIS-1489 CriticalAnalyzer Fault Injection Test 

Fault injection via Byteman to simulate deadlock and slow disk
sync scenarios, ensures the Critical Analyzer detects the issues
reacts appropriately.
This commit is contained in:
Martyn Taylor 2017-10-31 10:10:30 +00:00 committed by Clebert Suconic
parent 32802eeba1
commit 2e7b660ca8
4 changed files with 269 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/CriticalAnalyzerFaultInjectionTestBase.java Normal file
View File

@ -0,0 +1,91 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.activemq.artemis.tests.extras.byteman.critical.analyzer;
import javax.jms.DeliveryMode;
import javax.jms.MessageProducer;
import javax.jms.Queue;
import javax.jms.Session;
import org.apache.activemq.artemis.api.core.RoutingType;
import org.apache.activemq.artemis.api.core.SimpleString;
import org.apache.activemq.artemis.core.config.Configuration;
import org.apache.activemq.artemis.core.server.JournalType;
import org.apache.activemq.artemis.core.server.impl.AddressInfo;
import org.apache.activemq.artemis.tests.util.JMSTestBase;
import org.apache.activemq.artemis.tests.util.Wait;
import org.apache.activemq.artemis.utils.critical.CriticalAnalyzerPolicy;
import org.junit.Before;
public abstract class CriticalAnalyzerFaultInjectionTestBase extends JMSTestBase {
// Critical Analyzer Settings
private static long CHECK_PERIOD = 100;
private static long TIMEOUT = 3000;
private SimpleString address = SimpleString.toSimpleString("faultInjectionTestAddress");
private Thread t;
@Override
@Before
public void setUp() throws Exception {
super.setUp();
server.addAddressInfo(new AddressInfo(address, RoutingType.ANYCAST));
server.createQueue(address, RoutingType.ANYCAST, address, null, true, false);
conn = nettyCf.createConnection();
}
/*
Checks every 100ms timesout after 3000ms. Test should wait no longer than 3100s + Shutdown time.
*/
@Override
protected Configuration createDefaultConfig(boolean netty) throws Exception {
return super.createDefaultConfig(netty).setCriticalAnalyzerPolicy(CriticalAnalyzerPolicy.SHUTDOWN).setCriticalAnalyzer(true).setCriticalAnalyzerCheckPeriod(CHECK_PERIOD).setCriticalAnalyzerTimeout(TIMEOUT).setJournalType(JournalType.NIO);
}
@Override
public boolean usePersistence() {
return true;
}
void testSendDurableMessage() throws Exception {
try {
t = new Thread(() -> {
try {
Session s = conn.createSession(true, Session.SESSION_TRANSACTED);
Queue jmsQueue = s.createQueue(address.toString());
MessageProducer p = s.createProducer(jmsQueue);
p.setDeliveryMode(DeliveryMode.PERSISTENT);
conn.start();
for (int i = 0; i < 10; i++) {
p.send(s.createTextMessage("payload"));
}
s.commit();
} catch (Throwable e) {
e.printStackTrace();
}
});
t.start();
Wait.assertFalse(server::isStarted);
} finally {
t.interrupt();
}
}
}

81
tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/DeadlockStorageManagerTest.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.activemq.artemis.tests.extras.byteman.critical.analyzer;
import java.lang.reflect.Field;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.activemq.artemis.core.persistence.impl.journal.AbstractJournalStorageManager;
import org.jboss.byteman.contrib.bmunit.BMRule;
import org.jboss.byteman.contrib.bmunit.BMRules;
import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
import org.junit.Test;
import org.junit.runner.RunWith;
@RunWith(BMUnitRunner.class)
public class DeadlockStorageManagerTest extends CriticalAnalyzerFaultInjectionTestBase {
private static Thread lockT;
@BMRules(
rules = {
@BMRule(
name = "Deadlock on Storage Manager",
targetClass = "org.apache.activemq.artemis.core.persistence.impl.journal.AbstractJournalStorageManager",
targetMethod = "commit",
targetLocation = "ENTRY",
condition = "!flagged(\"testDeadlockOnStorageManager\")",
action = "org.apache.activemq.artemis.tests.extras.byteman.critical.analyzer.DeadlockStorageManagerTest.acquireStorageManagerLock($0);"),
@BMRule(
name = "Release Suspended Thread during Server Shutdown", // Releases wakes up suspended threads to allow shutdown to complete
targetClass = "org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl",
targetMethod = "stop",
targetLocation = "ENTRY",
action = "flag(\"testDeadlockOnStorageManager\"); org.apache.activemq.artemis.tests.extras.byteman.critical.analyzer.DeadlockStorageManagerTest.releaseStorageManagerLock()" )
})
@Test(timeout = 60000)
public void testDeadlockOnStorageManager() throws Exception {
testSendDurableMessage();
}
public static void acquireStorageManagerLock(final Object o) throws Exception {
// We're attempting to cause a deadlock on the readlock. This means we need to grab the write lock
// in a separate thread.
if (lockT == null) {
lockT = new Thread(() -> {
try {
Field field = AbstractJournalStorageManager.class.getDeclaredField("storageManagerLock");
field.setAccessible(true);
ReentrantReadWriteLock lock = ((ReentrantReadWriteLock) field.get(o));
lock.writeLock().lock();
try {
Thread.sleep(10000);
} catch (InterruptedException ie) {
lock.writeLock().unlock();
}
} catch (Exception e) {
fail();
}
});
lockT.start();
}
}
public static void releaseStorageManagerLock() throws NoSuchFieldException, IllegalAccessException {
lockT.interrupt();
}
}

48
tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/DeadlockedQueueTest.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.activemq.artemis.tests.extras.byteman.critical.analyzer;
import org.jboss.byteman.contrib.bmunit.BMRule;
import org.jboss.byteman.contrib.bmunit.BMRules;
import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
import org.junit.Test;
import org.junit.runner.RunWith;
@RunWith(BMUnitRunner.class)
public class DeadlockedQueueTest extends CriticalAnalyzerFaultInjectionTestBase {
@BMRules(
rules = {
@BMRule(
name = "Deadlock during Queue Delivery",
targetClass = "org.apache.activemq.artemis.core.server.impl.QueueImpl",
targetMethod = "deliver",
targetLocation = "AFTER SYNCHRONIZE",
condition = "!flagged(\"testDeadlockOnQueue\")",
action = "waitFor(\"testDeadlockOnQueue\")"),
@BMRule(
name = "Release Suspended Thread during Server Shutdown", // Releases wakes up suspended threads to allow shutdown to complete
targetClass = "org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl",
targetMethod = "stop",
targetLocation = "ENTRY",
action = "flag(\"testDeadlockOnQueue\"); signalWake(\"testDeadlockOnQueue\")")
})
@Test(timeout = 60000)
public void testDeadlockOnQueue() throws Exception {
testSendDurableMessage();
}
}

49
tests/extra-tests/src/test/java/org/apache/activemq/artemis/tests/extras/byteman/critical/analyzer/FileSystemSyncBlockedTest.java Normal file
View File

@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.activemq.artemis.tests.extras.byteman.critical.analyzer;
import org.jboss.byteman.contrib.bmunit.BMRule;
import org.jboss.byteman.contrib.bmunit.BMRules;
import org.jboss.byteman.contrib.bmunit.BMUnitRunner;
import org.junit.Test;
import org.junit.runner.RunWith;
@RunWith(BMUnitRunner.class)
public class FileSystemSyncBlockedTest extends CriticalAnalyzerFaultInjectionTestBase {
@BMRules(
rules = {
@BMRule(
name = "Simulate Slow Disk Sync",
targetClass = "org.apache.activemq.artemis.core.io.nio.NIOSequentialFile",
targetMethod = "sync",
targetLocation = "ENTRY",
condition = "!flagged(\"testSlowDiskSync\")", // Once the server shutdowns we stop applying this rule.
action = "waitFor(\"testSlowDiskSync\")"),
@BMRule(
// We ensure that no more
name = "Release Suspended Thread during Server Shutdown", // Releases wakes up suspended threads to allow shutdown to complete
targetClass = "org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl",
targetMethod = "stop",
targetLocation = "ENTRY",
action = "flag(\"testSlowDiskSync\"); signalWake(\"testSlowDiskSync\")")
})
@Test(timeout = 60000)
public void testSlowDiskSync() throws Exception {
testSendDurableMessage();
}
}