From 6a8cd175dc6276eee47792b6c45ca033d1b70539 Mon Sep 17 00:00:00 2001 From: Clebert Suconic Date: Thu, 24 Aug 2023 14:25:26 -0400 Subject: [PATCH] ARTEMIS-4401 improving JDBC Performance with Paging by a significant factor --- .../artemis/cli/commands/tools/PrintData.java | 4 +- .../cli/commands/etc/database-store.txt | 1 + .../config/ActiveMQDefaultConfiguration.java | 6 + .../store/drivers/AbstractJDBCDriver.java | 4 + .../store/drivers/JDBCConnectionProvider.java | 4 + .../store/drivers/JDBCDataSourceUtils.java | 2 +- .../jdbc/store/file/JDBCSequentialFile.java | 166 +- .../store/file/JDBCSequentialFileFactory.java | 24 +- .../file/JDBCSequentialFileFactoryDriver.java | 4 - .../jdbc/store/file/ScheduledWrite.java | 82 + .../file/JDBCSequentialFileFactoryTest.java | 17 +- .../core/io/SequentialFileFactory.java | 11 + .../artemis/maven/ArtemisAbstractPlugin.java | 28 +- .../artemis/maven/ArtemisCreatePlugin.java | 15 + .../maven/ArtemisDependencyScanPlugin.java | 21 +- .../core/config/StoreConfiguration.java | 4 + .../storage/DatabaseStorageConfiguration.java | 17 + .../impl/FileConfigurationParser.java | 1 + .../artemis/core/paging/PagingStore.java | 7 +- .../impl/PageCounterRebuildManager.java | 21 +- .../core/paging/impl/PageReadWriter.java | 12 +- .../impl/PagingStoreFactoryDatabase.java | 6 +- .../core/paging/impl/PagingStoreImpl.java | 43 +- .../core/persistence/StorageManager.java | 4 + .../journal/JDBCJournalStorageManager.java | 8 + .../impl/journal/JournalStorageManager.java | 9 + .../core/server/impl/ActiveMQServerImpl.java | 1 + .../schema/artemis-configuration.xsd | 8 + .../artemis/tests/util/ActiveMQTestBase.java | 14 +- docs/user-manual/paging.adoc | 5 + docs/user-manual/persistence.adoc | 16 +- pom.xml | 3 + tests/artemis-test-support/pom.xml | 6 + .../artemis/utils/RealServerTestBase.java | 272 +++ .../artemis/utils}/TestParameters.java | 5 +- tests/db-tests/README.md | 61 + tests/db-tests/jdbc-drivers/oracle/.gitignore | 1 + tests/db-tests/jdbc-drivers/oracle/README.md | 3 + tests/db-tests/pom.xml | 477 ++++ tests/db-tests/scripts/.gitignore | 2 + .../scripts/client-db2.sh} | 5 +- .../scripts/client-mssql.sh} | 5 +- .../scripts/client-mysql.sh} | 5 +- .../scripts/client-oracle.sh} | 7 +- tests/db-tests/scripts/client-postgres.sh | 21 + tests/db-tests/scripts/container-define.sh | 22 + .../scripts/db2.env} | 20 +- tests/db-tests/scripts/logs-db2.sh | 21 + tests/db-tests/scripts/logs-mssql.sh | 21 + tests/db-tests/scripts/logs-mysql.sh | 21 + tests/db-tests/scripts/logs-oracle.sh | 22 + tests/db-tests/scripts/logs-postgres.sh | 21 + .../scripts/print-license.sh} | 12 +- tests/db-tests/scripts/start-db2.sh | 60 + tests/db-tests/scripts/start-mssql.sh | 37 + tests/db-tests/scripts/start-mysql.sh | 22 + tests/db-tests/scripts/start-oracle.sh | 46 + tests/db-tests/scripts/start-postgres.sh | 22 + tests/db-tests/scripts/stop-all.sh | 23 + tests/db-tests/scripts/stop-db2.sh | 22 + tests/db-tests/scripts/stop-mssql.sh | 22 + tests/db-tests/scripts/stop-mysql.sh | 22 + tests/db-tests/scripts/stop-oracle.sh | 22 + tests/db-tests/scripts/stop-postgres.sh | 22 + .../activemq/artemis/tests/db/DropDBTest.java | 68 + .../artemis/tests/db/common/DBTestBase.java | 28 + .../artemis/tests/db/common/Database.java | 181 ++ .../tests/db/common/ParameterDBTestBase.java | 194 ++ .../db/invalid/JdbcStartupInvalidTest.java | 46 + .../tests/db/paging/GlobalPagingTest.java | 217 ++ .../db/paging/NetworkTimeoutCheckTest.java | 86 + .../artemis/tests/db/paging/PageSizeTest.java | 139 ++ .../artemis/tests/db}/paging/PagingTest.java | 2107 +++-------------- .../tests/db/paging/PrintDataTest.java | 103 + .../paging/RealServerDatabasePagingTest.java} | 49 +- .../tests/db/paging/SchemaValidationTest.java | 64 + ...Test.java => GlobalJournalPagingTest.java} | 31 +- .../integration/paging/JournalPagingTest.java | 762 ++++++ .../integration/paging/PagingOrderTest.java | 27 +- .../integration/paging/PagingSyncTest.java | 4 +- .../storage/PersistMultiThreadTest.java | 2 +- tests/pom.xml | 1 + tests/smoke-tests/pom.xml | 4 +- .../tests/smoke/common/SmokeTestBase.java | 177 +- tests/soak-tests/pom.xml | 77 - .../artemis/tests/soak/SoakTestBase.java | 244 +- .../tests/soak/client/ClientParameters.java | 2 +- .../soak/failover/RandomFailoverSoakTest.java | 2 +- .../artemis/tests/soak/owleak/OWLeakTest.java | 4 +- .../soak/paging/FlowControlPagingTest.java | 4 +- .../soak/paging/HorizontalPagingTest.java | 4 +- .../soak/paging/SubscriptionPagingTest.java | 4 +- .../core/paging/impl/PagingStoreImplTest.java | 7 +- 93 files changed, 4121 insertions(+), 2435 deletions(-) create mode 100644 artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/ScheduledWrite.java create mode 100644 tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/utils/RealServerTestBase.java rename tests/{soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak => artemis-test-support/src/main/java/org/apache/activemq/artemis/utils}/TestParameters.java (98%) create mode 100644 tests/db-tests/README.md create mode 100644 tests/db-tests/jdbc-drivers/oracle/.gitignore create mode 100644 tests/db-tests/jdbc-drivers/oracle/README.md create mode 100644 tests/db-tests/pom.xml create mode 100644 tests/db-tests/scripts/.gitignore rename tests/{soak-tests/src/test/scripts/stop-mysql-podman.sh => db-tests/scripts/client-db2.sh} (86%) rename tests/{soak-tests/src/test/scripts/client-mysql-podman.sh => db-tests/scripts/client-mssql.sh} (82%) rename tests/{soak-tests/src/test/scripts/stop-postgres-podman.sh => db-tests/scripts/client-mysql.sh} (86%) rename tests/{soak-tests/src/test/scripts/client-postgres-podman.sh => db-tests/scripts/client-oracle.sh} (77%) create mode 100755 tests/db-tests/scripts/client-postgres.sh create mode 100755 tests/db-tests/scripts/container-define.sh rename tests/{soak-tests/src/test/scripts/start-mysql-podman.sh => db-tests/scripts/db2.env} (73%) mode change 100755 => 100644 create mode 100755 tests/db-tests/scripts/logs-db2.sh create mode 100755 tests/db-tests/scripts/logs-mssql.sh create mode 100755 tests/db-tests/scripts/logs-mysql.sh create mode 100755 tests/db-tests/scripts/logs-oracle.sh create mode 100755 tests/db-tests/scripts/logs-postgres.sh rename tests/{soak-tests/src/test/scripts/start-postgres-podman.sh => db-tests/scripts/print-license.sh} (54%) create mode 100755 tests/db-tests/scripts/start-db2.sh create mode 100755 tests/db-tests/scripts/start-mssql.sh create mode 100755 tests/db-tests/scripts/start-mysql.sh create mode 100755 tests/db-tests/scripts/start-oracle.sh create mode 100755 tests/db-tests/scripts/start-postgres.sh create mode 100755 tests/db-tests/scripts/stop-all.sh create mode 100755 tests/db-tests/scripts/stop-db2.sh create mode 100755 tests/db-tests/scripts/stop-mssql.sh create mode 100755 tests/db-tests/scripts/stop-mysql.sh create mode 100755 tests/db-tests/scripts/stop-oracle.sh create mode 100755 tests/db-tests/scripts/stop-postgres.sh create mode 100644 tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/DropDBTest.java create mode 100644 tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/common/DBTestBase.java create mode 100644 tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/common/Database.java create mode 100644 tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/common/ParameterDBTestBase.java create mode 100644 tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/invalid/JdbcStartupInvalidTest.java create mode 100644 tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/GlobalPagingTest.java create mode 100644 tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/NetworkTimeoutCheckTest.java create mode 100644 tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/PageSizeTest.java rename tests/{integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration => db-tests/src/test/java/org/apache/activemq/artemis/tests/db}/paging/PagingTest.java (78%) create mode 100644 tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/PrintDataTest.java rename tests/{soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/DatabasePagingTest.java => db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/RealServerDatabasePagingTest.java} (74%) create mode 100644 tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/SchemaValidationTest.java rename tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/{GlobalPagingTest.java => GlobalJournalPagingTest.java} (92%) create mode 100644 tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/JournalPagingTest.java diff --git a/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/tools/PrintData.java b/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/tools/PrintData.java index 09e98d4808..1cdff63f3c 100644 --- a/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/tools/PrintData.java +++ b/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/tools/PrintData.java @@ -116,13 +116,13 @@ public class PrintData extends DBOption { printBanner(out, BINDINGS_BANNER); - DescribeJournal.printSurvivingRecords(storageManager.getBindingsJournal(), out, safe); + DescribeJournal bindings = DescribeJournal.printSurvivingRecords(storageManager.getBindingsJournal(), out, safe); printBanner(out, MESSAGES_BANNER); DescribeJournal describeJournal = DescribeJournal.printSurvivingRecords(storageManager.getMessageJournal(), out, safe); - printPages(describeJournal, storageManager, pagingmanager, out, safe, maxPages, null); + printPages(describeJournal, storageManager, pagingmanager, out, safe, maxPages, bindings); cleanup(); diff --git a/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/database-store.txt b/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/database-store.txt index 564c513a81..0b0cbff402 100644 --- a/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/database-store.txt +++ b/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/database-store.txt @@ -14,5 +14,6 @@ ${jdbcLockExpiration} ${jdbcLockRenewPeriod} ${jdbcNetworkTimeout} + 100K diff --git a/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/config/ActiveMQDefaultConfiguration.java b/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/config/ActiveMQDefaultConfiguration.java index 66231cfbb6..b7193d1988 100644 --- a/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/config/ActiveMQDefaultConfiguration.java +++ b/artemis-core-client/src/main/java/org/apache/activemq/artemis/api/config/ActiveMQDefaultConfiguration.java @@ -496,6 +496,8 @@ public final class ActiveMQDefaultConfiguration { private static final long DEFAULT_JDBC_ALLOWED_TIME_DIFF_MILLIS = 250; + private static final int DEFAULT_JDBC_MAX_PAGE_SIZE_BYTES = 100 * 1024; + // Default period to wait between connection TTL checks public static final long DEFAULT_CONNECTION_TTL_CHECK_INTERVAL = 2000; @@ -1512,6 +1514,10 @@ public final class ActiveMQDefaultConfiguration { return DEFAULT_JDBC_ALLOWED_TIME_DIFF_MILLIS; } + public static int getDefaultJdbcMaxPageSizeBytes() { + return DEFAULT_JDBC_MAX_PAGE_SIZE_BYTES; + } + public static long getDefaultConnectionTtlCheckInterval() { return DEFAULT_CONNECTION_TTL_CHECK_INTERVAL; } diff --git a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/drivers/AbstractJDBCDriver.java b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/drivers/AbstractJDBCDriver.java index 2e445e3b3e..03db79d6d2 100644 --- a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/drivers/AbstractJDBCDriver.java +++ b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/drivers/AbstractJDBCDriver.java @@ -184,6 +184,10 @@ public abstract class AbstractJDBCDriver { } } + public SQLProvider getSqlProvider() { + return sqlProvider; + } + public void setSqlProvider(SQLProvider sqlProvider) { this.sqlProvider = sqlProvider; } diff --git a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/drivers/JDBCConnectionProvider.java b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/drivers/JDBCConnectionProvider.java index afea96af49..95d250c03f 100644 --- a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/drivers/JDBCConnectionProvider.java +++ b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/drivers/JDBCConnectionProvider.java @@ -117,6 +117,10 @@ public class JDBCConnectionProvider { } } + public int getNetworkTimeoutMillis() { + return networkTimeoutMillis; + } + public void setNetworkTimeout(Executor executor, int milliseconds) { this.networkTimeoutExecutor = executor; this.networkTimeoutMillis = milliseconds; diff --git a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/drivers/JDBCDataSourceUtils.java b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/drivers/JDBCDataSourceUtils.java index 549fbe030e..d71e19b9ec 100644 --- a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/drivers/JDBCDataSourceUtils.java +++ b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/drivers/JDBCDataSourceUtils.java @@ -32,7 +32,7 @@ public class JDBCDataSourceUtils { .map(key -> key + "=" + (key.equalsIgnoreCase("password") ? "****" : dataSourceProperties.get(key))) .collect(Collectors.joining(", ", "{", "}"))); try { - DataSource dataSource = (DataSource) Class.forName(dataSourceClassName).newInstance(); + DataSource dataSource = (DataSource) Class.forName(dataSourceClassName).getDeclaredConstructor().newInstance(); for (Map.Entry entry : dataSourceProperties.entrySet()) { PropertyUtils.setProperty(dataSource, entry.getKey(), entry.getValue()); } diff --git a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFile.java b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFile.java index 96375af025..e2becc4e82 100644 --- a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFile.java +++ b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFile.java @@ -18,25 +18,32 @@ package org.apache.activemq.artemis.jdbc.store.file; import java.io.File; import java.io.IOException; +import java.lang.invoke.MethodHandles; import java.nio.ByteBuffer; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.activemq.artemis.api.core.ActiveMQBuffer; import org.apache.activemq.artemis.api.core.ActiveMQBuffers; import org.apache.activemq.artemis.api.core.ActiveMQException; import org.apache.activemq.artemis.api.core.ActiveMQExceptionType; +import org.apache.activemq.artemis.api.core.ActiveMQIOErrorException; import org.apache.activemq.artemis.core.io.IOCallback; import org.apache.activemq.artemis.core.io.SequentialFile; import org.apache.activemq.artemis.core.io.buffer.TimedBuffer; import org.apache.activemq.artemis.core.journal.EncodingSupport; import org.apache.activemq.artemis.core.journal.impl.SimpleWaitIOCallback; +import org.apache.activemq.artemis.core.server.ActiveMQScheduledComponent; +import org.apache.activemq.artemis.utils.ReusableLatch; +import org.jctools.queues.MpscUnboundedArrayQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; public class JDBCSequentialFile implements SequentialFile { @@ -64,12 +71,24 @@ public class JDBCSequentialFile implements SequentialFile { private final JDBCSequentialFileFactoryDriver dbDriver; + MpscUnboundedArrayQueue writeQueue = new MpscUnboundedArrayQueue<>(8192); + // Allows DB Drivers to cache meta data. private final Map metaData = new ConcurrentHashMap<>(); + final JDBCPageWriteScheduler pageWriteScheduler; + + final ScheduledExecutorService scheduledExecutorService; + + private final ReusableLatch pendingWrites = new ReusableLatch(); + + final long syncDelay; + JDBCSequentialFile(final JDBCSequentialFileFactory fileFactory, final String filename, final Executor executor, + final ScheduledExecutorService scheduledExecutorService, + final long syncDelay, final JDBCSequentialFileFactoryDriver driver, final Object writeLock) throws SQLException { this.fileFactory = fileFactory; @@ -78,6 +97,9 @@ public class JDBCSequentialFile implements SequentialFile { this.executor = executor; this.writeLock = writeLock; this.dbDriver = driver; + this.scheduledExecutorService = scheduledExecutorService; + this.syncDelay = syncDelay; + this.pageWriteScheduler = new JDBCPageWriteScheduler(scheduledExecutorService, executor, syncDelay); } void setWritePosition(long writePosition) { @@ -168,9 +190,9 @@ public class JDBCSequentialFile implements SequentialFile { } } - private synchronized int internalWrite(byte[] data, IOCallback callback, boolean append) { + private synchronized int jdbcWrite(byte[] data, IOCallback callback, boolean append) { try { - open(); + logger.debug("Writing {} bytes into {}", data.length, filename); synchronized (writeLock) { int noBytes = dbDriver.writeToFile(this, data, append); seek(append ? writePosition + noBytes : noBytes); @@ -189,52 +211,104 @@ public class JDBCSequentialFile implements SequentialFile { return 0; } - public synchronized int internalWrite(ActiveMQBuffer buffer, IOCallback callback) { - return internalWrite(buffer, callback, true); + public synchronized int jdbcWrite(ActiveMQBuffer buffer, IOCallback callback) { + return jdbcWrite(buffer, callback, true); } - public synchronized int internalWrite(ActiveMQBuffer buffer, IOCallback callback, boolean append) { + public synchronized int jdbcWrite(ActiveMQBuffer buffer, IOCallback callback, boolean append) { byte[] data = new byte[buffer.readableBytes()]; buffer.readBytes(data); - return internalWrite(data, callback, append); + return jdbcWrite(data, callback, append); } - private synchronized int internalWrite(ByteBuffer buffer, IOCallback callback) { - final byte[] data; - if (buffer.hasArray() && buffer.arrayOffset() == 0 && buffer.position() == 0 && buffer.limit() == buffer.array().length) { - data = buffer.array(); - } else { - byte[] copy = new byte[buffer.remaining()]; - buffer.get(copy); - data = copy; + + + private void pollWrites() { + if (writeQueue.isEmpty()) { + return; } - return internalWrite(data, callback, true); + + logger.debug("polling {} elements on {}", writeQueue.size(), this.filename); + + ArrayList writeList = new ArrayList<>(writeQueue.size()); // the size here is just an estimate + + byte[] bytes = extractBytes(writeList); + + jdbcWrite(bytes, null, true); + writeList.forEach(this::doCallback); + } + + /* Even though I would love to have a reusable byte[] for the following buffer + PreparedStatement.setData takes a byte[] without any sizing on the interface. + Blob interface would support setBytes with an offset and size, but some of the databases we are using + (DB2 specifically) is not allowing us to use Blob (at least during our dev time). + for that reason I'm using this byte[] with the very specific size that needs to be written + + Also Notice that our PagingManager will make sure that this size wouldn't go beyond our page-size limit + which we also limit at the JDBC storage, which should be 100K. */ + private byte[] extractBytes(ArrayList writeList) { + int totalSize = 0; + ScheduledWrite write; + while ((write = writeQueue.poll()) != null) { + writeList.add(write); + totalSize += write.readable(); + } + byte[] bytes = new byte[totalSize]; + + int writePosition = 0; + + for (ScheduledWrite el : writeList) { + writePosition += el.readAt(bytes, writePosition); + el.releaseBuffer(); + } + + return bytes; + } + + private void doCallback(ScheduledWrite write) { + if (write != null && write.callback != null) { + write.callback.done(); + } + pendingWrites.countDown(); } private void scheduleWrite(final ActiveMQBuffer bytes, final IOCallback callback, boolean append) { - executor.execute(() -> { - internalWrite(bytes, callback, append); - }); + scheduleWrite(new ScheduledWrite(bytes, callback, append)); + } + + private void scheduleWrite(ScheduledWrite scheduledWrite) { + logger.debug("offering {} bytes into {}", scheduledWrite.readable(), filename); + pendingWrites.countUp(); + writeQueue.offer(scheduledWrite); + this.pageWriteScheduler.delay(); } private void scheduleWrite(final ByteBuffer bytes, final IOCallback callback) { - executor.execute(() -> { - internalWrite(bytes, callback); - }); + scheduleWrite(new ScheduledWrite(bytes, callback, true)); } synchronized void seek(long noBytes) { writePosition = noBytes; } - public void write(ActiveMQBuffer bytes, boolean sync, IOCallback callback, boolean append) throws Exception { - // We ignore sync since we schedule writes straight away. + public void sendToDB(ActiveMQBuffer bytes, IOCallback callback, boolean append) throws Exception { + SimpleWaitIOCallback waitIOCallback = null; + + if (callback == null) { + waitIOCallback = new SimpleWaitIOCallback(); + callback = waitIOCallback; + } + scheduleWrite(bytes, callback, append); + + if (callback != null) { + waitIOCallback.waitCompletion(); + } } @Override public void write(ActiveMQBuffer bytes, boolean sync, IOCallback callback) throws Exception { - write(bytes, sync, callback, true); + sendToDB(bytes, callback, true); } @Override @@ -246,7 +320,7 @@ public class JDBCSequentialFile implements SequentialFile { public void write(EncodingSupport bytes, boolean sync, IOCallback callback) throws Exception { ActiveMQBuffer data = ActiveMQBuffers.fixedBuffer(bytes.getEncodeSize()); bytes.encode(data); - write(data, sync, callback, true); + sendToDB(data, callback, true); } @Override @@ -330,15 +404,27 @@ public class JDBCSequentialFile implements SequentialFile { fileFactory.sequentialFileClosed(this); } + public int getNetworkTimeoutMillis() { + return dbDriver.getJdbcConnectionProvider().getNetworkTimeoutMillis(); + } + @Override public void sync() throws IOException { - final SimpleWaitIOCallback callback = new SimpleWaitIOCallback(); - executor.execute(callback::done); - try { - callback.waitCompletion(); + int syncTimeout = getNetworkTimeoutMillis(); + + if (syncTimeout >= 0) { + if (!pendingWrites.await(syncTimeout, TimeUnit.MILLISECONDS)) { + fileFactory.onIOError(new ActiveMQIOErrorException("Database not responding to syncs before timeout"), "Error during JDBC file sync.", this); + } + } else { + // waiting forever however logger.debug while doing so + while (!pendingWrites.await(1, TimeUnit.SECONDS)) { + logger.debug("Awaiting syncs from database for page file {}", this.filename); + } + } + } catch (Exception e) { - callback.onError(ActiveMQExceptionType.IO_ERROR.getCode(), "Error during JDBC file sync."); fileFactory.onIOError(e, "Error during JDBC file sync.", this); } } @@ -363,7 +449,7 @@ public class JDBCSequentialFile implements SequentialFile { @Override public SequentialFile cloneFile() { try { - JDBCSequentialFile clone = new JDBCSequentialFile(fileFactory, filename, executor, dbDriver, writeLock); + JDBCSequentialFile clone = new JDBCSequentialFile(fileFactory, filename, executor, scheduledExecutorService, syncDelay, dbDriver, writeLock); clone.setWritePosition(this.writePosition); return clone; } catch (Exception e) { @@ -421,4 +507,20 @@ public class JDBCSequentialFile implements SequentialFile { public Object getMetaData(Object key) { return metaData.get(key); } + + + + private class JDBCPageWriteScheduler extends ActiveMQScheduledComponent { + + JDBCPageWriteScheduler(ScheduledExecutorService scheduledExecutorService, + Executor executor, + long checkPeriod) { + super(scheduledExecutorService, executor, checkPeriod, TimeUnit.MILLISECONDS, true); + } + + @Override + public void run() { + pollWrites(); + } + } } diff --git a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFileFactory.java b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFileFactory.java index e55fb4d03b..73b82f252f 100644 --- a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFileFactory.java +++ b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFileFactory.java @@ -36,6 +36,7 @@ import org.apache.activemq.artemis.utils.collections.ConcurrentHashSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.invoke.MethodHandles; +import java.util.concurrent.ScheduledExecutorService; public class JDBCSequentialFileFactory implements SequentialFileFactory, ActiveMQComponent { @@ -53,13 +54,22 @@ public class JDBCSequentialFileFactory implements SequentialFileFactory, ActiveM private final IOCriticalErrorListener criticalErrorListener; + private final long syncDelay; + + private final ScheduledExecutorService scheduledExecutorService; + + public JDBCSequentialFileFactory(final JDBCConnectionProvider connectionProvider, final SQLProvider sqlProvider, Executor executor, + ScheduledExecutorService scheduledExecutorService, + long syncDelay, IOCriticalErrorListener criticalErrorListener) throws Exception { this.executor = executor; this.criticalErrorListener = criticalErrorListener; + this.scheduledExecutorService = scheduledExecutorService; + this.syncDelay = syncDelay; try { this.dbDriver = JDBCFileUtils.getDBFileDriver(connectionProvider, sqlProvider); @@ -68,6 +78,11 @@ public class JDBCSequentialFileFactory implements SequentialFileFactory, ActiveM } } + @Override + public boolean supportsIndividualContext() { + return true; + } + public JDBCSequentialFileFactoryDriver getDbDriver() { return dbDriver; } @@ -114,7 +129,7 @@ public class JDBCSequentialFileFactory implements SequentialFileFactory, ActiveM public SequentialFile createSequentialFile(String fileName) { try { fileLocks.putIfAbsent(fileName, new Object()); - JDBCSequentialFile file = new JDBCSequentialFile(this, fileName, executor, dbDriver, fileLocks.get(fileName)); + JDBCSequentialFile file = new JDBCSequentialFile(this, fileName, executor, scheduledExecutorService, syncDelay, dbDriver, fileLocks.get(fileName)); files.add(file); return file; } catch (Exception e) { @@ -223,6 +238,13 @@ public class JDBCSequentialFileFactory implements SequentialFileFactory, ActiveM return null; } + + @Override + public String getDirectoryName() { + return dbDriver.getSqlProvider().getTableName(); + } + + @Override public boolean isStarted() { return started; diff --git a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFileFactoryDriver.java b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFileFactoryDriver.java index 4c1bccafc7..05726feabc 100644 --- a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFileFactoryDriver.java +++ b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFileFactoryDriver.java @@ -251,10 +251,6 @@ public class JDBCSequentialFileFactoryDriver extends AbstractJDBCDriver { } } - public int writeToFile(JDBCSequentialFile file, byte[] data) throws SQLException { - return writeToFile(file, data, true); - } - /** * Persists data to this files associated database mapping. * diff --git a/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/ScheduledWrite.java b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/ScheduledWrite.java new file mode 100644 index 0000000000..63ccce9bcf --- /dev/null +++ b/artemis-jdbc-store/src/main/java/org/apache/activemq/artemis/jdbc/store/file/ScheduledWrite.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.jdbc.store.file; + +import java.nio.ByteBuffer; + +import org.apache.activemq.artemis.api.core.ActiveMQBuffer; +import org.apache.activemq.artemis.core.io.IOCallback; + +public class ScheduledWrite { + ActiveMQBuffer amqBuffer; + ByteBuffer ioBuffer; + final IOCallback callback; + final boolean append; + + public ScheduledWrite(ActiveMQBuffer amqBuffer, IOCallback callback, boolean append) { + this.amqBuffer = amqBuffer; + this.ioBuffer = null; + this.callback = callback; + this.append = append; + } + + public ScheduledWrite(ByteBuffer ioBuffer, IOCallback callback, boolean append) { + this.ioBuffer = ioBuffer; + this.amqBuffer = null; + this.callback = callback; + this.append = append; + } + + // for a scheduled Callback without a write + public ScheduledWrite(IOCallback callback) { + this.ioBuffer = null; + this.amqBuffer = null; + this.callback = callback; + this.append = false; + } + + public int readable() { + if (ioBuffer != null) { + return ioBuffer.remaining(); + } else if (amqBuffer != null) { + return amqBuffer.readableBytes(); + } else { + return 0; + } + } + + public int readAt(byte[] dst, int offset) { + if (ioBuffer != null) { + int size = ioBuffer.remaining(); + ioBuffer.get(dst, offset, size); + return size; + } else if (amqBuffer != null) { + int size = amqBuffer.readableBytes(); + amqBuffer.getBytes(0, dst, offset, size); + return size; + } else { + return 0; + } + } + + /** Remove references letting buffer to be ready for GC */ + public void releaseBuffer() { + amqBuffer = null; + ioBuffer = null; + } +} diff --git a/artemis-jdbc-store/src/test/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFileFactoryTest.java b/artemis-jdbc-store/src/test/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFileFactoryTest.java index 485f30ee55..1dc22c936c 100644 --- a/artemis-jdbc-store/src/test/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFileFactoryTest.java +++ b/artemis-jdbc-store/src/test/java/org/apache/activemq/artemis/jdbc/store/file/JDBCSequentialFileFactoryTest.java @@ -31,6 +31,7 @@ import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.activemq.artemis.api.config.ActiveMQDefaultConfiguration; @@ -71,6 +72,8 @@ public class JDBCSequentialFileFactoryTest { private ExecutorService executor; + private ScheduledExecutorService scheduledExecutorService; + @Parameterized.Parameter public boolean useAuthentication; private String user = null; @@ -84,6 +87,7 @@ public class JDBCSequentialFileFactoryTest { @Before public void setup() throws Exception { executor = Executors.newSingleThreadExecutor(ActiveMQThreadFactory.defaultThreadFactory(getClass().getName())); + scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(ActiveMQThreadFactory.defaultThreadFactory(getClass().getName())); Map dataSourceProperties = new HashMap<>(); if (useAuthentication) { user = "testuser"; @@ -97,7 +101,7 @@ public class JDBCSequentialFileFactoryTest { dataSourceProperties.put("driverClassName", className); String tableName = "FILES"; String jdbcDatasourceClass = ActiveMQDefaultConfiguration.getDefaultDataSourceClassName(); - factory = new JDBCSequentialFileFactory(new JDBCConnectionProvider(JDBCDataSourceUtils.getDataSource(jdbcDatasourceClass, dataSourceProperties)), JDBCUtils.getSQLProvider(dataSourceProperties, tableName, SQLProvider.DatabaseStoreType.PAGE), executor, new IOCriticalErrorListener() { + factory = new JDBCSequentialFileFactory(new JDBCConnectionProvider(JDBCDataSourceUtils.getDataSource(jdbcDatasourceClass, dataSourceProperties)), JDBCUtils.getSQLProvider(dataSourceProperties, tableName, SQLProvider.DatabaseStoreType.PAGE), executor, scheduledExecutorService, 100, new IOCriticalErrorListener() { @Override public void onIOException(Throwable code, String message, String file) { } @@ -108,6 +112,7 @@ public class JDBCSequentialFileFactoryTest { @After public void tearDown() throws Exception { executor.shutdown(); + scheduledExecutorService.shutdown(); factory.destroy(); } @@ -231,7 +236,7 @@ public class JDBCSequentialFileFactoryTest { } IOCallbackCountdown callback = new IOCallbackCountdown(1); - file.internalWrite(src, callback); + file.jdbcWrite(src, callback); callback.assertEmpty(5); checkData(file, src); @@ -245,7 +250,7 @@ public class JDBCSequentialFileFactoryTest { ActiveMQBuffer src = ActiveMQBuffers.fixedBuffer(1); src.writeByte((byte)7); - file.internalWrite(src, null); + file.jdbcWrite(src, null); checkData(file, src); assertEquals(1, file.size()); file.close(); @@ -258,7 +263,7 @@ public class JDBCSequentialFileFactoryTest { for (int i = 0; i < bufferSize; i++) { src.writeByte((byte)i); } - file.internalWrite(src, null, false); + file.jdbcWrite(src, null, false); checkData(file, src); assertEquals(bufferSize, file.size()); } @@ -276,7 +281,7 @@ public class JDBCSequentialFileFactoryTest { } IOCallbackCountdown callback = new IOCallbackCountdown(1); - file.internalWrite(src, callback); + file.jdbcWrite(src, callback); JDBCSequentialFile copy = (JDBCSequentialFile) factory.createSequentialFile("copy.txt"); file.copyTo(copy); @@ -301,7 +306,7 @@ public class JDBCSequentialFileFactoryTest { } IOCallbackCountdown callback = new IOCallbackCountdown(1); - file.internalWrite(src, callback); + file.jdbcWrite(src, callback); assertEquals(bufferSize, file.size()); JDBCSequentialFile copy = (JDBCSequentialFile) file.cloneFile(); diff --git a/artemis-journal/src/main/java/org/apache/activemq/artemis/core/io/SequentialFileFactory.java b/artemis-journal/src/main/java/org/apache/activemq/artemis/core/io/SequentialFileFactory.java index d2f9c8e34e..fa4891e7a4 100644 --- a/artemis-journal/src/main/java/org/apache/activemq/artemis/core/io/SequentialFileFactory.java +++ b/artemis-journal/src/main/java/org/apache/activemq/artemis/core/io/SequentialFileFactory.java @@ -122,6 +122,10 @@ public interface SequentialFileFactory { File getDirectory(); + default String getDirectoryName() { + return getDirectory().getName(); + } + void clearBuffer(ByteBuffer buffer); void start(); @@ -140,4 +144,11 @@ public interface SequentialFileFactory { boolean isDatasync(); long getBufferSize(); + + /** Only JDBC supports individual context. + * Meaning for Files we need to use the Sync scheduler. + * for JDBC we need to use a callback from the JDBC completion thread to complete the IOContexts. */ + default boolean supportsIndividualContext() { + return false; + } } diff --git a/artemis-maven-plugin/src/main/java/org/apache/activemq/artemis/maven/ArtemisAbstractPlugin.java b/artemis-maven-plugin/src/main/java/org/apache/activemq/artemis/maven/ArtemisAbstractPlugin.java index 37803612e9..9b9ea0b664 100644 --- a/artemis-maven-plugin/src/main/java/org/apache/activemq/artemis/maven/ArtemisAbstractPlugin.java +++ b/artemis-maven-plugin/src/main/java/org/apache/activemq/artemis/maven/ArtemisAbstractPlugin.java @@ -22,6 +22,7 @@ import java.io.StringWriter; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -64,6 +65,11 @@ public abstract class ArtemisAbstractPlugin extends AbstractMojo { @Parameter(defaultValue = "${localRepository}") protected ArtifactRepository localRepository; + @Parameter + protected String[] extraRepositories; + + List inUseRepositories; + @Override public void execute() throws MojoExecutionException, MojoFailureException { if (isIgnore()) { @@ -90,8 +96,6 @@ public abstract class ArtemisAbstractPlugin extends AbstractMojo { protected abstract boolean isIgnore(); - protected abstract void doExecute() throws MojoExecutionException, MojoFailureException; - protected Artifact newArtifact(String artifactID) throws MojoFailureException { Artifact artifact; try { @@ -102,6 +106,26 @@ public abstract class ArtemisAbstractPlugin extends AbstractMojo { return artifact; } + protected void doExecute() throws MojoExecutionException, MojoFailureException { + int repositories = 0; + if (extraRepositories != null) { + inUseRepositories = new ArrayList<>(); + for (String strRepo: extraRepositories) { + RemoteRepository repo = new RemoteRepository.Builder("repo" + (repositories++), "default", strRepo).build(); + inUseRepositories.add(repo); + remoteRepos.add(repo); + } + } + } + + protected void done() { + if (inUseRepositories != null) { + inUseRepositories.forEach(r -> remoteRepos.remove(r)); + inUseRepositories.clear(); // give a help to GC + } + inUseRepositories = null; + } + protected File resolveArtifactFile(Artifact artifact) throws MojoExecutionException, DependencyCollectionException { ArtifactResult result = resolveArtifact(artifact); diff --git a/artemis-maven-plugin/src/main/java/org/apache/activemq/artemis/maven/ArtemisCreatePlugin.java b/artemis-maven-plugin/src/main/java/org/apache/activemq/artemis/maven/ArtemisCreatePlugin.java index ff05367623..77b3d6b2c1 100644 --- a/artemis-maven-plugin/src/main/java/org/apache/activemq/artemis/maven/ArtemisCreatePlugin.java +++ b/artemis-maven-plugin/src/main/java/org/apache/activemq/artemis/maven/ArtemisCreatePlugin.java @@ -141,6 +141,10 @@ public class ArtemisCreatePlugin extends ArtemisAbstractPlugin { @Parameter private String[] webListWithDeps; + /** Folders with libs to be copied into target */ + @Parameter() + private String[] libFolders; + @Parameter(defaultValue = "${localRepository}") private org.apache.maven.artifact.repository.ArtifactRepository localRepository; @@ -301,6 +305,17 @@ public class ArtemisCreatePlugin extends ArtemisAbstractPlugin { FileUtil.makeExec(commandLine); + if (libFolders != null) { + for (String libFolder : libFolders) { + File folder = new File(libFolder); + for (File file : folder.listFiles()) { + if (!file.isDirectory()) { + copyToDir("lib", file, commandLineStream); + } + } + } + } + if (getLog().isDebugEnabled()) { getLog().debug("###################################################################################################"); getLog().debug(commandLine.getName() + " created with commands to reproduce " + instance.getName()); diff --git a/artemis-maven-plugin/src/main/java/org/apache/activemq/artemis/maven/ArtemisDependencyScanPlugin.java b/artemis-maven-plugin/src/main/java/org/apache/activemq/artemis/maven/ArtemisDependencyScanPlugin.java index ba75866545..fc931d4e75 100644 --- a/artemis-maven-plugin/src/main/java/org/apache/activemq/artemis/maven/ArtemisDependencyScanPlugin.java +++ b/artemis-maven-plugin/src/main/java/org/apache/activemq/artemis/maven/ArtemisDependencyScanPlugin.java @@ -22,8 +22,6 @@ import java.io.FileOutputStream; import java.io.PrintStream; import java.nio.file.Files; import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Set; @@ -34,7 +32,6 @@ import org.apache.maven.plugins.annotations.LifecyclePhase; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; import org.apache.maven.project.MavenProject; -import org.eclipse.aether.repository.RemoteRepository; @Mojo(name = "dependency-scan", defaultPhase = LifecyclePhase.VERIFY) public class ArtemisDependencyScanPlugin extends ArtemisAbstractPlugin { @@ -53,9 +50,6 @@ public class ArtemisDependencyScanPlugin extends ArtemisAbstractPlugin { @Parameter private String[] libList; - @Parameter - private String[] extraRepositories; - @Parameter private String variableName; @@ -82,15 +76,8 @@ public class ArtemisDependencyScanPlugin extends ArtemisAbstractPlugin { @Override protected void doExecute() throws MojoExecutionException, MojoFailureException { - int repositories = 0; - List listRepo = new ArrayList<>(); - if (extraRepositories != null) { - for (String strRepo: extraRepositories) { - RemoteRepository repo = new RemoteRepository.Builder("repo" + (repositories++), "default", strRepo).build(); - listRepo.add(repo); - remoteRepos.add(repo); - } - } + super.doExecute(); + getLog().debug("Local " + localRepository); project = (MavenProject) getPluginContext().get("project"); @@ -160,9 +147,7 @@ public class ArtemisDependencyScanPlugin extends ArtemisAbstractPlugin { throw new MojoFailureException(e.getMessage()); } } finally { - for (RemoteRepository repository : listRepo) { - remoteRepos.remove(repository); - } + done(); } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/StoreConfiguration.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/StoreConfiguration.java index 553ece58b1..2cddb9792a 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/StoreConfiguration.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/StoreConfiguration.java @@ -26,4 +26,8 @@ public interface StoreConfiguration extends Serializable { } StoreType getStoreType(); + + default int getAllowedPageSize(int pageSize) { + return pageSize; + } } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/storage/DatabaseStorageConfiguration.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/storage/DatabaseStorageConfiguration.java index 51afacc98e..51fac53c6e 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/storage/DatabaseStorageConfiguration.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/config/storage/DatabaseStorageConfiguration.java @@ -69,6 +69,8 @@ public class DatabaseStorageConfiguration implements StoreConfiguration { private long jdbcAllowedTimeDiff = ActiveMQDefaultConfiguration.getDefaultJdbcAllowedTimeDiffMillis(); + private int maxPageSizeBytes = ActiveMQDefaultConfiguration.getDefaultJdbcMaxPageSizeBytes(); + @Override public StoreType getStoreType() { return StoreType.DATABASE; @@ -150,6 +152,15 @@ public class DatabaseStorageConfiguration implements StoreConfiguration { return jdbcAllowedTimeDiff; } + public int getMaxPageSizeBytes() { + return maxPageSizeBytes; + } + + public DatabaseStorageConfiguration setMaxPageSizeBytes(int maxPageSizeBytes) { + this.maxPageSizeBytes = maxPageSizeBytes; + return this; + } + /** * The DataSource to use to store Artemis data in the data store (can be {@code null} if {@code jdbcConnectionUrl} and {@code jdbcDriverClassName} are used instead). * @@ -306,4 +317,10 @@ public class DatabaseStorageConfiguration implements StoreConfiguration { public void setJdbcAllowedTimeDiff(long jdbcAllowedTimeDiff) { this.jdbcAllowedTimeDiff = jdbcAllowedTimeDiff; } + + @Override + public int getAllowedPageSize(int pageSize) { + return Math.min(pageSize, maxPageSizeBytes); + } + } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java index bbbbb22955..b4c05b6edc 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/deployers/impl/FileConfigurationParser.java @@ -2003,6 +2003,7 @@ public final class FileConfigurationParser extends XMLConfigurationUtil { conf.setJdbcLockExpirationMillis(getLong(storeNode, "jdbc-lock-expiration", conf.getJdbcLockExpirationMillis(), Validators.NO_CHECK)); conf.setJdbcJournalSyncPeriodMillis(getLong(storeNode, "jdbc-journal-sync-period", conf.getJdbcJournalSyncPeriodMillis(), Validators.NO_CHECK)); conf.setJdbcAllowedTimeDiff(getLong(storeNode, "jdbc-allowed-time-diff", conf.getJdbcAllowedTimeDiff(), Validators.NO_CHECK)); + conf.setMaxPageSizeBytes(getTextBytesAsIntBytes(storeNode, "jdbc-max-page-size-bytes", conf.getMaxPageSizeBytes(), Validators.NO_CHECK)); String jdbcUser = getString(storeNode, "jdbc-user", conf.getJdbcUser(), Validators.NO_CHECK); if (jdbcUser != null) { jdbcUser = PasswordMaskingUtil.resolveMask(mainConfig.isMaskPassword(), jdbcUser, mainConfig.getPasswordCodec()); diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/PagingStore.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/PagingStore.java index d582b0d712..74ff041b1b 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/PagingStore.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/PagingStore.java @@ -25,6 +25,7 @@ import org.apache.activemq.artemis.api.core.SimpleString; import org.apache.activemq.artemis.core.paging.cursor.PageCursorProvider; import org.apache.activemq.artemis.core.paging.cursor.PageSubscription; import org.apache.activemq.artemis.core.paging.impl.Page; +import org.apache.activemq.artemis.core.persistence.OperationContext; import org.apache.activemq.artemis.core.persistence.StorageManager; import org.apache.activemq.artemis.core.replication.ReplicationManager; import org.apache.activemq.artemis.core.server.ActiveMQComponent; @@ -60,6 +61,10 @@ public interface PagingStore extends ActiveMQComponent, RefCountMessageListener File getFolder(); + default String getFolderName() { + return getFolder().getName(); + } + AddressFullMessagePolicy getAddressFullMessagePolicy(); PageFullMessagePolicy getPageFullMessagePolicy(); @@ -99,7 +104,7 @@ public interface PagingStore extends ActiveMQComponent, RefCountMessageListener /** * Schedules sync to the file storage. */ - void sync() throws Exception; + void addSyncPoint(OperationContext context) throws Exception; /** * Performs a real sync on the current IO file. diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageCounterRebuildManager.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageCounterRebuildManager.java index b7218d65d7..62d40de398 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageCounterRebuildManager.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/cursor/impl/PageCounterRebuildManager.java @@ -75,7 +75,7 @@ public class PageCounterRebuildManager implements Runnable { try { paging = store.isPaging(); if (!paging) { - logger.debug("Destination {} was not paging, no need to rebuild counters"); + logger.trace("Destination {} was not paging, no need to rebuild counters"); store.getCursorProvider().forEachSubscription(subscription -> { subscription.getCounter().markRebuilding(); subscription.getCounter().finishRebuild(); @@ -89,7 +89,7 @@ public class PageCounterRebuildManager implements Runnable { limitPageId = store.getCurrentWritingPage(); limitMessageNr = currentPage.getNumberOfMessages(); if (logger.isDebugEnabled()) { - logger.debug("PageCounterRebuild for {}, Current writing page {} and limit will be {} with lastMessage on last page={}", store.getStoreName(), store.getCurrentWritingPage(), limitPageId, limitMessageNr); + logger.trace("PageCounterRebuild for {}, Current writing page {} and limit will be {} with lastMessage on last page={}", store.getStoreName(), store.getCurrentWritingPage(), limitPageId, limitMessageNr); } } catch (Exception e) { logger.warn(e.getMessage(), e); @@ -196,25 +196,26 @@ public class PageCounterRebuildManager implements Runnable { public void rebuild() throws Exception { if (pgStore == null) { - logger.debug("Page store is null during rebuildCounters"); + logger.trace("Page store is null during rebuildCounters"); return; } if (!paging) { - logger.debug("Ignoring call to rebuild pgStore {}", pgStore.getAddress()); + logger.trace("Ignoring call to rebuild pgStore {}", pgStore.getAddress()); } - logger.debug("Rebuilding counter for store {}", pgStore.getAddress()); + logger.debug("Rebuilding page counter for address {}", pgStore.getAddress()); for (long pgid = pgStore.getFirstPage(); pgid <= limitPageId; pgid++) { if (logger.isDebugEnabled()) { - logger.debug("Rebuilding counter on messages from page {} on rebuildCounters for address {}", pgid, pgStore.getAddress()); + logger.trace("Rebuilding counter on messages from page {} on rebuildCounters for address {}", pgid, pgStore.getAddress()); } + logger.debug("{} reading paging {} of {}", pgStore.getAddress(), pgid, limitPageId); Page page = pgStore.newPageObject(pgid); if (!page.getFile().exists()) { if (logger.isDebugEnabled()) { - logger.debug("Skipping page {} on store {}", pgid, pgStore.getAddress()); + logger.trace("Skipping page {} on store {}", pgid, pgStore.getAddress()); } continue; } @@ -227,14 +228,14 @@ public class PageCounterRebuildManager implements Runnable { PagedMessage msg = iter.next(); if (storedLargeMessages != null && msg.getMessage().isLargeMessage()) { if (logger.isDebugEnabled()) { - logger.debug("removing storedLargeMessage {}", msg.getMessage().getMessageID()); + logger.trace("removing storedLargeMessage {}", msg.getMessage().getMessageID()); } storedLargeMessages.remove(msg.getMessage().getMessageID()); } if (limitPageId == pgid) { if (msg.getMessageNumber() >= limitMessageNr) { if (logger.isDebugEnabled()) { - logger.debug("Rebuild counting on {} reached the last message at {}-{}", pgStore.getAddress(), limitPageId, limitMessageNr); + logger.trace("Rebuild counting on {} reached the last message at {}-{}", pgStore.getAddress(), limitPageId, limitMessageNr); } // this is the limit where we should count.. // anything beyond this will be new data @@ -286,7 +287,7 @@ public class PageCounterRebuildManager implements Runnable { boolean txIncluded = msg.getTransactionID() <= 0 || transactions == null || txInfo != null; if (!txIncluded) { - logger.debug("TX is not included for {}", msg); + logger.trace("TX is not included for {}", msg); } if (ok && txIncluded) { // not acked and TX is ok diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/impl/PageReadWriter.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/impl/PageReadWriter.java index e006086fa0..4a6fa3da42 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/impl/PageReadWriter.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/impl/PageReadWriter.java @@ -27,7 +27,9 @@ import org.apache.activemq.artemis.core.buffers.impl.ChannelBufferWrapper; import org.apache.activemq.artemis.core.io.SequentialFile; import org.apache.activemq.artemis.core.io.SequentialFileFactory; import org.apache.activemq.artemis.core.paging.PagedMessage; +import org.apache.activemq.artemis.core.persistence.OperationContext; import org.apache.activemq.artemis.core.persistence.StorageManager; +import org.apache.activemq.artemis.core.persistence.impl.journal.OperationContextImpl; import org.apache.activemq.artemis.core.server.LargeServerMessage; import org.apache.activemq.artemis.utils.DataConstants; import org.apache.activemq.artemis.utils.Env; @@ -83,7 +85,15 @@ public class PageReadWriter { assert (activeMQBuffer.readableBytes() == bufferSize) : "messageEncodedSize is different from expected"; //buffer limit and position are the same assert (buffer.remaining() == bufferSize) : "buffer position or limit are changed"; - file.writeDirect(buffer, false); + if (fileFactory.supportsIndividualContext()) { + OperationContext context = OperationContextImpl.getContext(); + if (context != null) { + context.storeLineUp(); + } + file.writeDirect(buffer, false, context); + } else { + file.writeDirect(buffer, false); + } return bufferSize; } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/impl/PagingStoreFactoryDatabase.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/impl/PagingStoreFactoryDatabase.java index d55e175e51..2300a132f1 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/impl/PagingStoreFactoryDatabase.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/impl/PagingStoreFactoryDatabase.java @@ -124,7 +124,7 @@ public class PagingStoreFactoryDatabase implements PagingStoreFactory { if (sqlProviderFactory == null) { sqlProviderFactory = new PropertySQLProvider.Factory(dbConf.getConnectionProvider()); } - pagingFactoryFileFactory = new JDBCSequentialFileFactory(dbConf.getConnectionProvider(), sqlProviderFactory.create(pageStoreTableNamePrefix, SQLProvider.DatabaseStoreType.PAGE), executorFactory.getExecutor(), criticalErrorListener); + pagingFactoryFileFactory = new JDBCSequentialFileFactory(dbConf.getConnectionProvider(), sqlProviderFactory.create(pageStoreTableNamePrefix, SQLProvider.DatabaseStoreType.PAGE), executorFactory.getExecutor(), scheduledExecutor, dbConf.getJdbcJournalSyncPeriodMillis(), criticalErrorListener); pagingFactoryFileFactory.start(); started = true; } @@ -193,7 +193,7 @@ public class PagingStoreFactoryDatabase implements PagingStoreFactory { } } - directoryList.write(writeBuffer, true, null, false); + directoryList.sendToDB(writeBuffer, null, false); directoryList.close(); } } @@ -258,7 +258,7 @@ public class PagingStoreFactoryDatabase implements PagingStoreFactory { sqlProviderFactory = new PropertySQLProvider.Factory(dbConf.getConnectionProvider()); } sqlProvider = sqlProviderFactory.create(getTableNameForGUID(directoryName), SQLProvider.DatabaseStoreType.PAGE); - final JDBCSequentialFileFactory fileFactory = new JDBCSequentialFileFactory(dbConf.getConnectionProvider(), sqlProvider, executorFactory.getExecutor(), criticalErrorListener); + final JDBCSequentialFileFactory fileFactory = new JDBCSequentialFileFactory(dbConf.getConnectionProvider(), sqlProvider, executorFactory.getExecutor(), scheduledExecutor, dbConf.getJdbcJournalSyncPeriodMillis(), criticalErrorListener); factoryToTableName.put(fileFactory, directoryName); return fileFactory; } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/impl/PagingStoreImpl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/impl/PagingStoreImpl.java index 675b85ee9d..6d332099b0 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/impl/PagingStoreImpl.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/paging/impl/PagingStoreImpl.java @@ -43,6 +43,7 @@ import org.apache.activemq.artemis.core.paging.PagingStore; import org.apache.activemq.artemis.core.paging.PagingStoreFactory; import org.apache.activemq.artemis.core.paging.cursor.PageCursorProvider; import org.apache.activemq.artemis.core.paging.cursor.PageSubscription; +import org.apache.activemq.artemis.core.persistence.OperationContext; import org.apache.activemq.artemis.core.persistence.StorageManager; import org.apache.activemq.artemis.core.replication.ReplicationManager; import org.apache.activemq.artemis.core.server.ActiveMQMessageBundle; @@ -232,7 +233,9 @@ public class PagingStoreImpl implements PagingStore { configureSizeMetric(); - pageSize = addressSettings.getPageSizeBytes(); + // JDBC has a maximum page size of 100K by default. + // it can be reconfigured through jdbc-max-page-size-bytes in the JDBC configuration section + pageSize = storageManager.getAllowedPageSize(addressSettings.getPageSizeBytes()); addressFullMessagePolicy = addressSettings.getAddressFullMessagePolicy(); @@ -430,6 +433,11 @@ public class PagingStoreImpl implements PagingStore { } } + @Override + public String getFolderName() { + return fileFactory.getDirectoryName(); + } + @Override public boolean isPaging() { AddressFullMessagePolicy policy = this.addressFullMessagePolicy; @@ -461,26 +469,29 @@ public class PagingStoreImpl implements PagingStore { } @Override - public void sync() throws Exception { - if (syncTimer != null) { - syncTimer.addSync(storageManager.getContext()); - } else { - ioSync(); + public void addSyncPoint(OperationContext context) throws Exception { + if (fileFactory == null || !fileFactory.supportsIndividualContext()) { + if (syncTimer != null) { + syncTimer.addSync(context); + } else { + ioSync(); + } } - } @Override public void ioSync() throws Exception { - lock.readLock().lock(); + if (!fileFactory.supportsIndividualContext()) { + lock.readLock().lock(); - try { - final Page page = currentPage; - if (page != null) { - page.sync(); + try { + final Page page = currentPage; + if (page != null) { + page.sync(); + } + } finally { + lock.readLock().unlock(); } - } finally { - lock.readLock().unlock(); } } @@ -1214,7 +1225,7 @@ public class PagingStoreImpl implements PagingStore { page.write(pagedMessage); if (tx == null && syncNonTransactional && message.isDurable()) { - sync(); + addSyncPoint(storageManager.getContext()); } if (logger.isTraceEnabled()) { @@ -1384,7 +1395,7 @@ public class PagingStoreImpl implements PagingStore { */ private void syncStore() throws Exception { for (PagingStore store : usedStores) { - store.sync(); + store.addSyncPoint(storageManager.getContext()); } } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/persistence/StorageManager.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/persistence/StorageManager.java index 8e43f4ed72..022ae4ccb8 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/persistence/StorageManager.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/persistence/StorageManager.java @@ -499,4 +499,8 @@ public interface StorageManager extends IDGenerator, ActiveMQComponent { void persistIdGenerator(); void injectMonitor(FileStoreMonitor monitor) throws Exception; + + default int getAllowedPageSize(int pageSize) { + return pageSize; + } } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/persistence/impl/journal/JDBCJournalStorageManager.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/persistence/impl/journal/JDBCJournalStorageManager.java index 1ab90c0af7..a0a2035435 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/persistence/impl/journal/JDBCJournalStorageManager.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/persistence/impl/journal/JDBCJournalStorageManager.java @@ -16,6 +16,7 @@ */ package org.apache.activemq.artemis.core.persistence.impl.journal; +import java.lang.invoke.MethodHandles; import java.nio.ByteBuffer; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledExecutorService; @@ -32,9 +33,13 @@ import org.apache.activemq.artemis.jdbc.store.sql.PropertySQLProvider; import org.apache.activemq.artemis.jdbc.store.sql.SQLProvider; import org.apache.activemq.artemis.utils.ExecutorFactory; import org.apache.activemq.artemis.utils.critical.CriticalAnalyzer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class JDBCJournalStorageManager extends JournalStorageManager { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public JDBCJournalStorageManager(Configuration config, CriticalAnalyzer analyzer, ExecutorFactory executorFactory, @@ -80,12 +85,15 @@ public class JDBCJournalStorageManager extends JournalStorageManager { connectionProvider, sqlProviderFactory.create(dbConf.getLargeMessageTableName(), SQLProvider.DatabaseStoreType.LARGE_MESSAGE), executorFactory.getExecutor(), + scheduledExecutorService, + dbConf.getJdbcJournalSyncPeriodMillis(), criticalErrorListener); this.bindingsJournal = bindingsJournal; this.messageJournal = messageJournal; this.largeMessagesFactory = largeMessagesFactory; largeMessagesFactory.start(); } catch (Exception e) { + logger.warn(e.getMessage(), e); criticalErrorListener.onIOException(e, e.getMessage(), null); } } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/persistence/impl/journal/JournalStorageManager.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/persistence/impl/journal/JournalStorageManager.java index 336a1820df..82ce95be56 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/persistence/impl/journal/JournalStorageManager.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/persistence/impl/journal/JournalStorageManager.java @@ -895,4 +895,13 @@ public class JournalStorageManager extends AbstractJournalStorageManager { monitor.addStore(bindingsFF.getDirectory()); } } + + @Override + public int getAllowedPageSize(int pageSize) { + if (config.getStoreConfiguration() == null) { + return pageSize; + } + return config.getStoreConfiguration().getAllowedPageSize(pageSize); + } + } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ActiveMQServerImpl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ActiveMQServerImpl.java index e1b8ad6b86..efc3fe2d32 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ActiveMQServerImpl.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ActiveMQServerImpl.java @@ -554,6 +554,7 @@ public class ActiveMQServerImpl implements ActiveMQServer { private void configureJdbcNetworkTimeout() { if (configuration.isPersistenceEnabled()) { if (configuration.getStoreConfiguration() != null && configuration.getStoreConfiguration().getStoreType() == StoreConfiguration.StoreType.DATABASE) { + configuration.setMaxDiskUsage(-1); // it does not make sense with JDBC DatabaseStorageConfiguration databaseStorageConfiguration = (DatabaseStorageConfiguration) configuration.getStoreConfiguration(); databaseStorageConfiguration.setConnectionProviderNetworkTimeout(threadPool, databaseStorageConfiguration.getJdbcNetworkTimeout()); } diff --git a/artemis-server/src/main/resources/schema/artemis-configuration.xsd b/artemis-server/src/main/resources/schema/artemis-configuration.xsd index 62ffa946a7..58274341dc 100644 --- a/artemis-server/src/main/resources/schema/artemis-configuration.xsd +++ b/artemis-server/src/main/resources/schema/artemis-configuration.xsd @@ -2786,6 +2786,14 @@ + + + + The max page size (in bytes) to use for all addresses when using JDBC. + Supports byte notation like "K", "Mb", "MiB", "GB", etc. + + + diff --git a/artemis-server/src/test/java/org/apache/activemq/artemis/tests/util/ActiveMQTestBase.java b/artemis-server/src/test/java/org/apache/activemq/artemis/tests/util/ActiveMQTestBase.java index 5537efcac0..6e32ca69e1 100644 --- a/artemis-server/src/test/java/org/apache/activemq/artemis/tests/util/ActiveMQTestBase.java +++ b/artemis-server/src/test/java/org/apache/activemq/artemis/tests/util/ActiveMQTestBase.java @@ -332,7 +332,7 @@ public abstract class ActiveMQTestBase extends Assert { private static final String EXPECTED_DERBY_SHUTDOWN_STATE = "XJ015"; /** This method will be passed as a lambda into runAfter from createDefaultDatabaseStorageConfiguration */ - protected final void dropDerby() throws Exception { + protected void dropDerby() throws Exception { String user = getJDBCUser(); String password = getJDBCPassword(); try { @@ -926,11 +926,11 @@ public abstract class ActiveMQTestBase extends Assert { return "memory:" + getTestDir(); } - protected final String getTestJDBCConnectionUrl() { + protected String getTestJDBCConnectionUrl() { return System.getProperty("jdbc.connection.url", "jdbc:derby:" + getEmbeddedDataBaseName() + ";create=true"); } - protected final String getJDBCClassName() { + protected String getJDBCClassName() { return System.getProperty("jdbc.driver.class", "org.apache.derby.jdbc.EmbeddedDriver"); } @@ -1213,15 +1213,11 @@ public abstract class ActiveMQTestBase extends Assert { * @throws InterruptedException */ protected void waitForNotPaging(Queue queue) throws InterruptedException { - waitForNotPaging(queue.getPageSubscription().getPagingStore()); + waitForNotPaging(queue.getPagingStore()); } protected void waitForNotPaging(PagingStore store) throws InterruptedException { - long timeout = System.currentTimeMillis() + 20000; - while (timeout > System.currentTimeMillis() && store.isPaging()) { - Thread.sleep(100); - } - assertFalse(store.isPaging()); + Wait.assertFalse("Store is still paging", store::isPaging, 20_000); } protected static Topology waitForTopology(final ActiveMQServer server, final int nodes) throws Exception { diff --git a/docs/user-manual/paging.adoc b/docs/user-manual/paging.adoc index 713fe4d74b..ac817ec6e5 100644 --- a/docs/user-manual/paging.adoc +++ b/docs/user-manual/paging.adoc @@ -123,6 +123,11 @@ This tells what to do if the system is reaching `page-limit-bytes` or `page-limi | |=== + +[NOTE] + +When using the JDBC storage, the effective page-size-bytes used is limited to jdbc-max-page-size-bytes, configured in the JDBC storage section. + === max-size-bytes and max-size-messages simultaneous usage It is possible to define max-size-messages (as the maximum number of messages) and max-messages-size (as the max number of estimated memory used by the address) concurrently. diff --git a/docs/user-manual/persistence.adoc b/docs/user-manual/persistence.adoc index ab77d47341..d1c489ca8c 100644 --- a/docs/user-manual/persistence.adoc +++ b/docs/user-manual/persistence.adoc @@ -343,11 +343,14 @@ Performance for both paging and large messages is especially diminished with JDB The JDBC persistence layer is targeted to those users who _must_ use a database e.g. due to internal company policy. ==== -ActiveMQ Artemis currently has support for a limited number of database vendors (older versions may work but mileage may vary): +ActiveMQ Artemis currently has support for a limited number of database vendors: -. PostgreSQL 9.4.x -. MySQL 5.7.x -. Apache Derby 10.11.1.1 +. PostgreSQL +. MySQL +. Microsoft SQL Server +. Oracle +. DB2 +. Apache Derby The JDBC store uses a JDBC connection to store messages and bindings data in records in database tables. The data stored in the database tables is encoded using Apache ActiveMQ Artemis internal encodings. @@ -428,6 +431,10 @@ The maximal time offset between the broker and the database in milliseconds when Currently this value only affects the logging and will show a warning if the detected difference exceeds the limit. The default value is 250 milliseconds. +jdbc-max-page-size-bytes:: +The maximal size a page can use. The default and recommended maximum value is 100K bytes. +Using larger sizes will result in downloading large blobs that would affect performance when using paged messages. + NOTE: Some DBMS (e.g. Oracle, 30 chars) have restrictions on the size of table names, this should be taken into consideration when configuring table names for the Artemis database store, pay particular attention to the page store table name, which can be appended with a unique ID of up to 20 characters. (for Oracle this would mean configuring a page-store-table-name of max size of 10 chars). @@ -446,6 +453,7 @@ It is also possible to explicitly add the user and password rather than in the J MESSAGE_TABLE LARGE_MESSAGES_TABLE NODE_MANAGER_TABLE + 100K ---- diff --git a/pom.xml b/pom.xml index c34fe140e6..e18879cf4b 100644 --- a/pom.xml +++ b/pom.xml @@ -1816,6 +1816,9 @@ examples/protocols/amqp/dotnet/**/bin/**/* examples/protocols/amqp/dotnet/**/readme.md examples/protocols/amqp/**/readme.md + + + tests/db-tests/jdbc-drivers/**/* diff --git a/tests/artemis-test-support/pom.xml b/tests/artemis-test-support/pom.xml index 113f051cec..6c3fe6aab3 100644 --- a/tests/artemis-test-support/pom.xml +++ b/tests/artemis-test-support/pom.xml @@ -43,6 +43,12 @@ ${project.version} provided + + org.apache.activemq + artemis-cli + ${project.version} + provided + org.apache.activemq artemis-jms-client diff --git a/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/utils/RealServerTestBase.java b/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/utils/RealServerTestBase.java new file mode 100644 index 0000000000..7cf9f0353b --- /dev/null +++ b/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/utils/RealServerTestBase.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.utils; + +import javax.management.MBeanServerInvocationHandler; +import javax.management.ObjectName; +import javax.management.remote.JMXConnector; +import javax.management.remote.JMXConnectorFactory; +import javax.management.remote.JMXServiceURL; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.net.MalformedURLException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.activemq.artemis.api.core.RoutingType; +import org.apache.activemq.artemis.api.core.SimpleString; +import org.apache.activemq.artemis.api.core.management.ActiveMQServerControl; +import org.apache.activemq.artemis.api.core.management.ObjectNameBuilder; +import org.apache.activemq.artemis.api.core.management.QueueControl; +import org.apache.activemq.artemis.api.jms.ActiveMQJMSClient; +import org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory; +import org.apache.activemq.artemis.tests.util.ActiveMQTestBase; +import org.apache.activemq.artemis.util.ServerUtil; +import org.junit.After; +import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// Base class for tests that will start a real server +public class RealServerTestBase extends ActiveMQTestBase { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String STOP_FILE_NAME = "STOP_ME"; + + Set processes = new HashSet<>(); + private static final String JMX_SERVER_HOSTNAME = "localhost"; + private static final int JMX_SERVER_PORT = 10099; + + public static final String basedir = System.getProperty("basedir"); + + @After + public void after() throws Exception { + // close ServerLocators before killing the server otherwise they'll hang and delay test termination + closeAllServerLocatorsFactories(); + + for (Process process : processes) { + try { + ServerUtil.killServer(process, true); + } catch (Throwable e) { + e.printStackTrace(); + } + } + processes.clear(); + } + + public void killServer(Process process) { + processes.remove(process); + try { + ServerUtil.killServer(process); + } catch (Throwable e) { + e.printStackTrace(); + } + } + + protected static void stopServerWithFile(String serverLocation) throws IOException { + File serverPlace = new File(serverLocation); + File etcPlace = new File(serverPlace, "etc"); + File stopMe = new File(etcPlace, STOP_FILE_NAME); + Assert.assertTrue(stopMe.createNewFile()); + } + + public static String getServerLocation(String serverName) { + return basedir + "/target/" + serverName; + } + + public static boolean cleanupData(String serverName) { + String location = getServerLocation(serverName); + boolean result = deleteDirectory(new File(location, "data")); + deleteDirectory(new File(location, "log")); + return result; + } + + public void addProcess(Process process) { + processes.add(process); + } + + public void removeProcess(Process process) { + processes.remove(process); + } + + public Process startServer(String serverName, int portID, int timeout) throws Exception { + Process process = ServerUtil.startServer(getServerLocation(serverName), serverName, portID, timeout); + addProcess(process); + return process; + } + + public Process startServer(String serverName, String uri, int timeout) throws Exception { + Process process = ServerUtil.startServer(getServerLocation(serverName), serverName, uri, timeout); + addProcess(process); + return process; + } + + protected JMXConnector getJmxConnector() throws MalformedURLException { + return getJmxConnector(JMX_SERVER_HOSTNAME, JMX_SERVER_PORT); + } + + protected static JMXConnector newJMXFactory(String uri) throws Throwable { + return JMXConnectorFactory.connect(new JMXServiceURL(uri)); + } + + protected static ActiveMQServerControl getServerControl(String uri, + ObjectNameBuilder builder, + long timeout) throws Throwable { + long expireLoop = System.currentTimeMillis() + timeout; + Throwable lastException = null; + do { + try { + JMXConnector connector = newJMXFactory(uri); + + ActiveMQServerControl serverControl = MBeanServerInvocationHandler.newProxyInstance(connector.getMBeanServerConnection(), builder.getActiveMQServerObjectName(), ActiveMQServerControl.class, false); + serverControl.isActive(); // making one call to make sure it's working + return serverControl; + } catch (Throwable e) { + lastException = e; + Thread.sleep(500); + } + } + while (expireLoop > System.currentTimeMillis()); + + throw lastException; + } + + protected static JMXConnector getJmxConnector(String hostname, int port) throws MalformedURLException { + // Without this, the RMI server would bind to the default interface IP (the user's local IP mostly) + System.setProperty("java.rmi.server.hostname", hostname); + + // I don't specify both ports here manually on purpose. See actual RMI registry connection port extraction below. + String urlString = "service:jmx:rmi:///jndi/rmi://" + hostname + ":" + port + "/jmxrmi"; + + JMXServiceURL url = new JMXServiceURL(urlString); + JMXConnector jmxConnector = null; + + try { + jmxConnector = JMXConnectorFactory.connect(url); + System.out.println("Successfully connected to: " + urlString); + } catch (Exception e) { + jmxConnector = null; + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + return jmxConnector; + } + + protected static final void recreateBrokerDirectory(final String homeInstance) { + recreateDirectory(homeInstance + "/data"); + recreateDirectory(homeInstance + "/log"); + } + + protected void checkLogRecord(File logFile, boolean exist, String... values) throws Exception { + Assert.assertTrue(logFile.exists()); + boolean hasRecord = false; + try (BufferedReader reader = new BufferedReader(new FileReader(logFile))) { + String line = reader.readLine(); + while (line != null) { + if (line.contains(values[0])) { + boolean hasAll = true; + for (int i = 1; i < values.length; i++) { + if (!line.contains(values[i])) { + hasAll = false; + break; + } + } + if (hasAll) { + hasRecord = true; + System.out.println("audit has it: " + line); + break; + } + } + line = reader.readLine(); + } + if (exist) { + Assert.assertTrue(hasRecord); + } else { + Assert.assertFalse(hasRecord); + } + } + } + + protected static QueueControl getQueueControl(String uri, + ObjectNameBuilder builder, + String address, + String queueName, + RoutingType routingType, + long timeout) throws Throwable { + long expireLoop = System.currentTimeMillis() + timeout; + Throwable lastException = null; + do { + try { + JMXConnector connector = newJMXFactory(uri); + + ObjectName objectQueueName = builder.getQueueObjectName(SimpleString.toSimpleString(address), SimpleString.toSimpleString(queueName), routingType); + + QueueControl queueControl = MBeanServerInvocationHandler.newProxyInstance(connector.getMBeanServerConnection(), objectQueueName, QueueControl.class, false); + queueControl.getMessagesAcknowledged(); // making one call + return queueControl; + } catch (Throwable e) { + logger.warn(e.getMessage(), e); + lastException = e; + Thread.sleep(500); + } + } + while (expireLoop > System.currentTimeMillis()); + + throw lastException; + } + + + protected static void unzip(File zipFile, File serverFolder) throws IOException, ClassNotFoundException, InterruptedException { + ProcessBuilder zipBuilder = new ProcessBuilder("unzip", zipFile.getAbsolutePath()).directory(serverFolder); + + Process process = zipBuilder.start(); + SpawnedVMSupport.startLogger("zip", process); + logger.info("Zip finished with {}", process.waitFor()); + } + + + protected static void zip(File zipFile, File serverFolder) throws IOException, ClassNotFoundException, InterruptedException { + logger.info("Zipping data folder for {}", zipFile); + ProcessBuilder zipBuilder = new ProcessBuilder("zip", "-r", zipFile.getAbsolutePath(), "data").directory(serverFolder); + Process process = zipBuilder.start(); + SpawnedVMSupport.startLogger("zip", process); + logger.info("Zip finished with {}", process.waitFor()); + } + + public boolean waitForServerToStart(String uri, String username, String password, long timeout) throws InterruptedException { + long realTimeout = System.currentTimeMillis() + timeout; + while (System.currentTimeMillis() < realTimeout) { + try (ActiveMQConnectionFactory cf = ActiveMQJMSClient.createConnectionFactory(uri, null)) { + cf.createConnection(username, password).close(); + System.out.println("server " + uri + " started"); + } catch (Exception e) { + System.out.println("awaiting server " + uri + " start at "); + Thread.sleep(500); + continue; + } + return true; + } + + return false; + } + +} diff --git a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/TestParameters.java b/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/utils/TestParameters.java similarity index 98% rename from tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/TestParameters.java rename to tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/utils/TestParameters.java index 32494da1a9..0ac7989605 100644 --- a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/TestParameters.java +++ b/tests/artemis-test-support/src/main/java/org/apache/activemq/artemis/utils/TestParameters.java @@ -15,12 +15,13 @@ * limitations under the License. */ -package org.apache.activemq.artemis.tests.soak; +package org.apache.activemq.artemis.utils; + +import java.lang.invoke.MethodHandles; import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; /** Encapsulates System properties that could be passed on to the test. */ public class TestParameters { diff --git a/tests/db-tests/README.md b/tests/db-tests/README.md new file mode 100644 index 0000000000..54f548c0e8 --- /dev/null +++ b/tests/db-tests/README.md @@ -0,0 +1,61 @@ +# Database Tests + +This module runs tests against selected Databases. + +There is one profile for each supported Database: + +- DB-derby-tests +- DB-postgres-tests +- DB-mysql-tests +- DB-mssql-tests +- DB-db2-tests +- DB-oracle-tests + +To enable the testsuite to run against any of these databases, simply enable the corresponding profiles. + +## Providing Databases + +When enabling a Database profile, you must download and provide the running database for that profile. You can also configure the JDBC URI. + +You can refer to the examples provided under `./scripts`. Please note that you are responsible for complying with the licensing requirements of each database you provide. + +## Configuring the JDBC URI + +You can pass the JDBC URI as a parameter using the following supported parameters: + +- `derby.uri` +- `postgres.uri` +- `mysql.uri` +- `mssql.uri` +- `oracle.uri` +- `db2.uri` + +Example: + +```shell +mvn -Pdb2.uri='jdbc:db2://MyDB2Server:50000/artemis:user=db2inst1;password=artemis;' +``` + +#Security Authorization +Tests on this module will perform several `drop table` and `create table` operations. So the user used to connecto these databases must have such `security grants`. +Also It is recommended to the database schema allocated exclusively to this testsuite. + +# Servers + +One Artemis server is created for each supported database. After building, they will be available under ./target/${DATABASE}: + +- `./target/derby` +- `./target/postgres` +- `./target/mysql` +- `./target/mssql` +- `./target/db2` + +Some of the tests on this module are connecting to the database directly, and these tests will use the JDBC jar directly from the `lib folder` from each of these servers. + +# Oracle JDBC Driver + +All the JDBC drivers using in this module are available as maven central repository and are being downloaded by the artemis-maven-plugin during the compilation phase of the tests. + +The exception to this rule is [Oracle database](https://www.oracle.com/database/technologies/appdev/jdbc-downloads.html), for which you must provide the JAR under this location: + +- jdbc-drivers/oracle \ No newline at end of file diff --git a/tests/db-tests/jdbc-drivers/oracle/.gitignore b/tests/db-tests/jdbc-drivers/oracle/.gitignore new file mode 100644 index 0000000000..d392f0e82c --- /dev/null +++ b/tests/db-tests/jdbc-drivers/oracle/.gitignore @@ -0,0 +1 @@ +*.jar diff --git a/tests/db-tests/jdbc-drivers/oracle/README.md b/tests/db-tests/jdbc-drivers/oracle/README.md new file mode 100644 index 0000000000..5893b786ce --- /dev/null +++ b/tests/db-tests/jdbc-drivers/oracle/README.md @@ -0,0 +1,3 @@ +Add the oracle jdbc driver that you downloaded from oracle.com and own a license on this folder. so the create process will place it in the correct place. + +At the time of the creation of this file, the JDBC driver for Oracle could be found at https://www.oracle.com/database/technologies/appdev/jdbc-downloads.html diff --git a/tests/db-tests/pom.xml b/tests/db-tests/pom.xml new file mode 100644 index 0000000000..d8a3088edf --- /dev/null +++ b/tests/db-tests/pom.xml @@ -0,0 +1,477 @@ + + + 4.0.0 + + org.apache.activemq.tests + artemis-tests-pom + 2.31.0-SNAPSHOT + + + db-tests + jar + Database Tests + + + ${project.basedir}/../../ + + -Dderby.load=${derby.load} -Dpostgres.load=${postgres.load} -Dpostgres.uri=${postgres.uri} + -Dpostgres.class=${postgres.class} -Dmssql.load=${mssql.load} -Dmssql.uri=${mssql.uri} + -Dmssql.class=${mssql.class} -Dmysql.load=${mysql.load} -Dmysql.uri=${mysql.uri} -Dmysql.class=${mysql.class} + -Ddb2.load=${db2.load} -Ddb2.class=${db2.class} -Ddb2.uri=${db2.uri} -Doracle.load=${oracle.load} + -Doracle.uri=${oracle.uri} -Doracle.class=${oracle.class} + + + -Ddistribution.lib="${activemq.basedir}/artemis-distribution/target/apache-artemis-${project.version}-bin/apache-artemis-${project.version}/lib" + + + false + + false + jdbc:db2://localhost:50000/artemis:user=db2inst1;password=artemis; + com.ibm.db2.jcc.DB2Driver + + false + jdbc:mysql://localhost/ARTEMIS-TEST?user=root&#38;password=artemis + com.mysql.cj.jdbc.Driver + + false + jdbc:postgresql://localhost:5432/artemis?user=artemis&#38;password=artemis + org.postgresql.Driver + + false + jdbc:oracle:thin:system/artemis@localhost:1521:FREE + oracle.jdbc.driver.OracleDriver + + false + jdbc:sqlserver://localhost:1433;user=sa;password=ActiveMQ*Artemis + com.microsoft.sqlserver.jdbc.SQLServerDriver + + true + + + + + + org.apache.activemq + apache-artemis + ${project.version} + compile + pom + + + org.apache.activemq + artemis-server + ${project.version} + test + + + org.apache.activemq.tests + artemis-test-support + ${project.version} + test + + + org.apache.activemq + artemis-core-client + ${project.version} + test + + + org.apache.activemq + artemis-jms-client + ${project.version} + test + + + org.apache.activemq + artemis-commons + ${project.version} + test + + + org.apache.activemq + artemis-cli + ${project.version} + test + + + org.apache.activemq + artemis-journal + ${project.version} + test + + + junit + junit + test + + + io.netty + netty-common + test + + + org.jctools + jctools-core + test + + + org.apache.qpid + qpid-jms-client + test + + + + org.slf4j + slf4j-api + test + + + org.apache.logging.log4j + log4j-slf4j-impl + test + + + jakarta.jms + jakarta.jms-api + test + + + jakarta.management.j2ee + jakarta.management.j2ee-api + test + + + + + org.apache.johnzon + johnzon-core + test + + + jakarta.json + jakarta.json-api + test + + + + org.apache.activemq + artemis-unit-test-support + ${project.version} + test + + + + + org.apache.derby + derby + test + + + + + + + + + org.apache.activemq + artemis-maven-plugin + + + test-compile + create-jdbc-bad-driver + + create + + + amq + admin + admin + false + true + ${basedir}/target/jdbc-bad-driver + + --shared-store + --jdbc + --jdbc-connection-url + tcp://noexist + --jdbc-driver-class-name + badDriver + + + + + test-compile + create-derby + + create + + + + -Djava.net.preferIPv4Stack=true + ${basedir}/target/derby + ${basedir}/target/classes/servers/derby + true + + org.apache.derby:derby:${apache.derby.version} + + + --jdbc + --global-max-messages + 100 + --java-options + -ea + + + + + test-compile + create-mysql + + create + + + + -Djava.net.preferIPv4Stack=true + ${basedir}/target/mysql + ${basedir}/target/classes/servers/mysql + true + + com.mysql:mysql-connector-j:8.0.33 + + + --jdbc + --jdbc-connection-url + ${mysql.uri} + --jdbc-driver-class-name + ${mysql.class} + --global-max-messages + 100 + --java-options + -ea + + + + + test-compile + create-postgres + + create + + + + -Djava.net.preferIPv4Stack=true + ${basedir}/target/postgres + ${basedir}/target/classes/servers/postgres + true + + org.postgresql:postgresql:42.6.0 + + + --jdbc + --jdbc-connection-url + ${postgres.uri} + --jdbc-driver-class-name + ${postgres.class} + --global-max-messages + 100 + --java-options + -ea + + + + + test-compile + create-oracle + + create + + + + -Djava.net.preferIPv4Stack=true + ${basedir}/target/oracle + true + + + ${project.basedir}/jdbc-drivers/oracle + + + --jdbc + --jdbc-connection-url + ${oracle.uri} + --jdbc-driver-class-name + ${oracle.class} + --global-max-messages + 100 + --java-options + -ea + + + + + test-compile + create-mssql + + create + + + + com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre11 + + + -Djava.net.preferIPv4Stack=true + ${basedir}/target/mssql + true + + --jdbc + --jdbc-connection-url + ${mssql.uri} + --jdbc-driver-class-name + ${mssql.class} + --global-max-messages + 100 + --java-options + -ea + + + + + test-compile + create-DB2 + + create + + + + com.ibm.db2:jcc:11.5.8.0 + + + + -Djava.net.preferIPv4Stack=true + ${basedir}/target/db2 + true + + --jdbc + --jdbc-connection-url + ${db2.uri} + --jdbc-driver-class-name + ${db2.class} + --global-max-messages + 100 + --java-options + -ea + + + + + + + org.apache.activemq.tests + smoke-tests + ${project.version} + + + + + org.apache.maven.plugins + maven-surefire-plugin + + 1 + false + ${skipDBTests} + ${db-extra-args} ${activemq-surefire-argline} ${artemis-distribution-lib-dir} + + ${postgres.uri} + + + + + + + + + DB-all-tests + + false + true + true + true + true + true + + + + + DB-oracle-tests + + false + true + + + + + DB-mysql-tests + + true + false + + + + + DB-mssql-tests + + true + false + + + + + DB-db2-tests + + true + false + + + + + DB-postgres-tests + + true + false + + + + + DB-derby-tests + + true + false + + + + + diff --git a/tests/db-tests/scripts/.gitignore b/tests/db-tests/scripts/.gitignore new file mode 100644 index 0000000000..e3fc730364 --- /dev/null +++ b/tests/db-tests/scripts/.gitignore @@ -0,0 +1,2 @@ +../../soak-tests/src/test/scripts/oradb +db2db diff --git a/tests/soak-tests/src/test/scripts/stop-mysql-podman.sh b/tests/db-tests/scripts/client-db2.sh similarity index 86% rename from tests/soak-tests/src/test/scripts/stop-mysql-podman.sh rename to tests/db-tests/scripts/client-db2.sh index e23a1159b8..84a21f7855 100755 --- a/tests/soak-tests/src/test/scripts/stop-mysql-podman.sh +++ b/tests/db-tests/scripts/client-db2.sh @@ -16,7 +16,6 @@ # specific language governing permissions and limitations # under the License. -# This script shows a simple way to stop a mysql with podman +source ./container-define.sh -podman kill mysql-artemis-test -podman rm mysql-artemis-test \ No newline at end of file +$CONTAINER_COMMAND exec -it db2-artemis-test bash diff --git a/tests/soak-tests/src/test/scripts/client-mysql-podman.sh b/tests/db-tests/scripts/client-mssql.sh similarity index 82% rename from tests/soak-tests/src/test/scripts/client-mysql-podman.sh rename to tests/db-tests/scripts/client-mssql.sh index bd37ee2455..71dbcd276e 100755 --- a/tests/soak-tests/src/test/scripts/client-mysql-podman.sh +++ b/tests/db-tests/scripts/client-mssql.sh @@ -16,5 +16,6 @@ # specific language governing permissions and limitations # under the License. -# Start a command line mysql for the Database started with ./start-mysql-podman.sh -podman exec -it mysql-artemis-test mysql ARTEMIS-TEST -u root --password=artemis +source ./container-define.sh + +$CONTAINER_COMMAND exec -it mssql-artemis-test /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P ActiveMQ*Artemis \ No newline at end of file diff --git a/tests/soak-tests/src/test/scripts/stop-postgres-podman.sh b/tests/db-tests/scripts/client-mysql.sh similarity index 86% rename from tests/soak-tests/src/test/scripts/stop-postgres-podman.sh rename to tests/db-tests/scripts/client-mysql.sh index 298fbac33c..2cca1c3763 100755 --- a/tests/soak-tests/src/test/scripts/stop-postgres-podman.sh +++ b/tests/db-tests/scripts/client-mysql.sh @@ -16,7 +16,6 @@ # specific language governing permissions and limitations # under the License. -# This script shows a simple way to stop a mysql with podman +source ./container-define.sh -podman kill postgres-artemis-test -podman rm postgres-artemis-test \ No newline at end of file +$CONTAINER_COMMAND exec -it mysql-artemis-test mysql ARTEMIS-TEST -u root --password=artemis diff --git a/tests/soak-tests/src/test/scripts/client-postgres-podman.sh b/tests/db-tests/scripts/client-oracle.sh similarity index 77% rename from tests/soak-tests/src/test/scripts/client-postgres-podman.sh rename to tests/db-tests/scripts/client-oracle.sh index 5c0a745e63..71db1a0102 100755 --- a/tests/soak-tests/src/test/scripts/client-postgres-podman.sh +++ b/tests/db-tests/scripts/client-oracle.sh @@ -16,6 +16,7 @@ # specific language governing permissions and limitations # under the License. -# Start a command line mysql for the Database started with ./start-postgres-podman.sh -podman exec -it postgres-artemis-test psql -U artemis artemis -#podman exec -it mysql-artemis-test mysql ARTEMIS-TEST -u root --password=artemis +source ./container-define.sh + +$CONTAINER_COMMAND exec -it oracle-artemis-test sqlplus system/artemis@FREE + diff --git a/tests/db-tests/scripts/client-postgres.sh b/tests/db-tests/scripts/client-postgres.sh new file mode 100755 index 0000000000..0d46535214 --- /dev/null +++ b/tests/db-tests/scripts/client-postgres.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +$CONTAINER_COMMAND exec -it postgres-artemis-test psql -U artemis artemis \ No newline at end of file diff --git a/tests/db-tests/scripts/container-define.sh b/tests/db-tests/scripts/container-define.sh new file mode 100755 index 0000000000..5a4273981d --- /dev/null +++ b/tests/db-tests/scripts/container-define.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#CONTAINER_COMMAND=$CONTAINER_COMMAND + +# you may use 'podman' or 'docker' +CONTAINER_COMMAND=podman diff --git a/tests/soak-tests/src/test/scripts/start-mysql-podman.sh b/tests/db-tests/scripts/db2.env old mode 100755 new mode 100644 similarity index 73% rename from tests/soak-tests/src/test/scripts/start-mysql-podman.sh rename to tests/db-tests/scripts/db2.env index 28ad4157f7..b4ee3e70f5 --- a/tests/soak-tests/src/test/scripts/start-mysql-podman.sh +++ b/tests/db-tests/scripts/db2.env @@ -1,4 +1,3 @@ -#!/bin/sh # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,7 +15,18 @@ # specific language governing permissions and limitations # under the License. -# This script shows a simple way to start a mysql with podman - -./stop-mysql-podman.sh -podman run -d -p 3306:3306 --name mysql-artemis-test --rm -e MYSQL_ROOT_PASSWORD=artemis -e MYSQL_USER=artemis -e MYSQL_PASSWORD=artemis -e MYSQL_DATABASE=ARTEMIS-TEST mysql:8 \ No newline at end of file +DB2INSTANCE=db2inst1 +DB2INST1_PASSWORD=artemis +DBNAME=artemis +BLU=false +ENABLE_ORACLE_COMPATIBILITY=false +UPDATEAVAIL=NO +TO_CREATE_SAMPLEDB=false +REPODB=false +IS_OSXFS=false +PERSISTENT_HOME=false +HADR_ENABLED=false +ETCD_ENDPOINT= +ETCD_USERNAME= +ETCD_PASSWORD= +AUTOCONFIG=false diff --git a/tests/db-tests/scripts/logs-db2.sh b/tests/db-tests/scripts/logs-db2.sh new file mode 100755 index 0000000000..80f5fb15ba --- /dev/null +++ b/tests/db-tests/scripts/logs-db2.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +$CONTAINER_COMMAND logs -f db2-artemis-test diff --git a/tests/db-tests/scripts/logs-mssql.sh b/tests/db-tests/scripts/logs-mssql.sh new file mode 100755 index 0000000000..c5cb00e810 --- /dev/null +++ b/tests/db-tests/scripts/logs-mssql.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +$CONTAINER_COMMAND logs -f mssql-artemis-test \ No newline at end of file diff --git a/tests/db-tests/scripts/logs-mysql.sh b/tests/db-tests/scripts/logs-mysql.sh new file mode 100755 index 0000000000..320d938e87 --- /dev/null +++ b/tests/db-tests/scripts/logs-mysql.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +$CONTAINER_COMMAND logs -f mysql-artemis-test diff --git a/tests/db-tests/scripts/logs-oracle.sh b/tests/db-tests/scripts/logs-oracle.sh new file mode 100755 index 0000000000..17d174a1da --- /dev/null +++ b/tests/db-tests/scripts/logs-oracle.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +$CONTAINER_COMMAND logs -f oracle-artemis-test + diff --git a/tests/db-tests/scripts/logs-postgres.sh b/tests/db-tests/scripts/logs-postgres.sh new file mode 100755 index 0000000000..3e63894547 --- /dev/null +++ b/tests/db-tests/scripts/logs-postgres.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +$CONTAINER_COMMAND logs -f postgres-artemis-test \ No newline at end of file diff --git a/tests/soak-tests/src/test/scripts/start-postgres-podman.sh b/tests/db-tests/scripts/print-license.sh similarity index 54% rename from tests/soak-tests/src/test/scripts/start-postgres-podman.sh rename to tests/db-tests/scripts/print-license.sh index b18d966423..8d902f07d1 100755 --- a/tests/soak-tests/src/test/scripts/start-postgres-podman.sh +++ b/tests/db-tests/scripts/print-license.sh @@ -16,7 +16,13 @@ # specific language governing permissions and limitations # under the License. -# This script shows a simple way to start a mysql with podman +source ./container-define.sh -./stop-postgres-podman.sh -podman run --name postgres-artemis-test --rm -d -e POSTGRES_USER=artemis -e POSTGRES_PASSWORD=artemis -e POSTGRES_DB=artemis -p 5432:5432 -p 9876:80 postgres +echo "" +echo "*******************************************************************************************************************************" +echo " Notice:" +echo " This script is provided solely to assist you in running a $1 Free Database in a $CONTAINER_COMMAND environment," +echo " to facilitate development and testing, using an image provided by $2." +echo " By running this script, you agree to abide by all licensing terms issued by $2 for the $1 image being downloaded here." +echo "*******************************************************************************************************************************" +echo "" diff --git a/tests/db-tests/scripts/start-db2.sh b/tests/db-tests/scripts/start-db2.sh new file mode 100755 index 0000000000..cbc20225e1 --- /dev/null +++ b/tests/db-tests/scripts/start-db2.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + + +# NOTE: at the time this script was written podman had an issue starting DB2 without a folder specified. Docker ran it without any problems. +# If you must use podman you could specify a data folder and it should work fine + +# As documented on https://www.ibm.com/docs/en/db2/11.5?topic=system-linux + +export LICENSE=reject + +if [ $# -ge 1 ]; then + # Check if the first argument is --accept-license + if [ "$1" == "--accept-license" ]; then + ./print-license.sh "DB2" "IBM" + export LICENSE=accept + else + echo "Warning: you must accept the DB2 license. Run ./logs-db2.sh to check the log output." + echo "Usage: $0 --accept-license " + fi +else + echo "Warning: you must accept the DB2 license. Run ./logs-db2.sh to check the log output." + echo "Usage: $0 --accept-license " +fi + +if [ $# -ne 2 ]; then + echo "NO_DATA has been specified. not using a data folder" + data_argument="" + folder_data="NO_DATA" +else + folder_data=$2 + data_argument="-v $folder_data:/database:Z" +fi + +./stop-db2.sh + +if [ ! -d "$folder_data" ] && [ "$folder_data" != "NO_DATA" ]; then + mkdir "$folder_data" + chmod 777 $folder_data + echo "Folder '$folder_data' created." +fi + +$CONTAINER_COMMAND run -d -h db2-artemis-test --name db2-artemis-test --privileged=true -p 50000:50000 -eLICENSE=$LICENSE --env-file db2.env $data_argument icr.io/db2_community/db2 \ No newline at end of file diff --git a/tests/db-tests/scripts/start-mssql.sh b/tests/db-tests/scripts/start-mssql.sh new file mode 100755 index 0000000000..e5783a7f2a --- /dev/null +++ b/tests/db-tests/scripts/start-mssql.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +export LICENSE="ACCEPT_EULA=N" + +if [ $# -ge 1 ]; then + # Check if the first argument is --accept-license + if [ "$1" == "--accept-license" ]; then + ./print-license.sh "SQL Server" "Microsoft" + export LICENSE="ACCEPT_EULA=Y" + else + echo "Warning: you must accept the Microsoft license. Run ./logs-mssql.sh to check the log output." + fi +else + echo "Warning: you must accept the Microsoft license. Run ./logs-mssql.sh to check the log output." +fi + +./stop-mssql.sh + +$CONTAINER_COMMAND run -d --name mssql-artemis-test -e "$LICENSE" -e "MSSQL_SA_PASSWORD=ActiveMQ*Artemis" -p 1433:1433 mcr.microsoft.com/mssql/server:2019-latest diff --git a/tests/db-tests/scripts/start-mysql.sh b/tests/db-tests/scripts/start-mysql.sh new file mode 100755 index 0000000000..e5dce31e2f --- /dev/null +++ b/tests/db-tests/scripts/start-mysql.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +./stop-mysql.sh +$CONTAINER_COMMAND run -d -p 3306:3306 --name mysql-artemis-test --rm -e MYSQL_ROOT_PASSWORD=artemis -e MYSQL_USER=artemis -e MYSQL_PASSWORD=artemis -e MYSQL_DATABASE=ARTEMIS-TEST mysql:8 \ No newline at end of file diff --git a/tests/db-tests/scripts/start-oracle.sh b/tests/db-tests/scripts/start-oracle.sh new file mode 100755 index 0000000000..0922b0c9e9 --- /dev/null +++ b/tests/db-tests/scripts/start-oracle.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + echo " setting folder_data as NO_DATA by default" + folder_data="NO_DATA" +else + folder_data="$1" +fi + +./stop-oracle.sh + +./print-license.sh Oracle Oracle + +if [ "$folder_data" = "NO_DATA" ]; then + echo "NO_DATA has been specified. not using a data folder" + data_argument="" +else + data_argument="-v $folder_data:/opt/oracle/oradata:Z" +fi + +if [ ! -d "$folder_data" ] && [ "$folder_data" != "NO_DATA" ]; then + mkdir "$folder_data" + chmod 777 $folder_data + echo "Folder '$folder_data' created." +fi + +$CONTAINER_COMMAND run -d --name oracle-artemis-test -p 1521:1521 $data_argument -e ORACLE_PWD=artemis container-registry.oracle.com/database/free:latest diff --git a/tests/db-tests/scripts/start-postgres.sh b/tests/db-tests/scripts/start-postgres.sh new file mode 100755 index 0000000000..4c6e1d230f --- /dev/null +++ b/tests/db-tests/scripts/start-postgres.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +./stop-postgres.sh +$CONTAINER_COMMAND run --name postgres-artemis-test --rm -d -e POSTGRES_USER=artemis -e POSTGRES_PASSWORD=artemis -e POSTGRES_DB=artemis -p 5432:5432 -p 9876:80 postgres diff --git a/tests/db-tests/scripts/stop-all.sh b/tests/db-tests/scripts/stop-all.sh new file mode 100755 index 0000000000..0d62f83317 --- /dev/null +++ b/tests/db-tests/scripts/stop-all.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +./stop-db2.sh +./stop-mssql.sh +./stop-mysql.sh +./stop-oracle.sh +./stop-postgres.sh diff --git a/tests/db-tests/scripts/stop-db2.sh b/tests/db-tests/scripts/stop-db2.sh new file mode 100755 index 0000000000..0c823a3376 --- /dev/null +++ b/tests/db-tests/scripts/stop-db2.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +$CONTAINER_COMMAND kill db2-artemis-test +$CONTAINER_COMMAND rm -f db2-artemis-test diff --git a/tests/db-tests/scripts/stop-mssql.sh b/tests/db-tests/scripts/stop-mssql.sh new file mode 100755 index 0000000000..e113d81c7d --- /dev/null +++ b/tests/db-tests/scripts/stop-mssql.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +$CONTAINER_COMMAND kill mssql-artemis-test +$CONTAINER_COMMAND rm mssql-artemis-test \ No newline at end of file diff --git a/tests/db-tests/scripts/stop-mysql.sh b/tests/db-tests/scripts/stop-mysql.sh new file mode 100755 index 0000000000..e371bf6daa --- /dev/null +++ b/tests/db-tests/scripts/stop-mysql.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +$CONTAINER_COMMAND kill mysql-artemis-test +$CONTAINER_COMMAND rm mysql-artemis-test \ No newline at end of file diff --git a/tests/db-tests/scripts/stop-oracle.sh b/tests/db-tests/scripts/stop-oracle.sh new file mode 100755 index 0000000000..82325e28d1 --- /dev/null +++ b/tests/db-tests/scripts/stop-oracle.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +$CONTAINER_COMMAND kill oracle-artemis-test +$CONTAINER_COMMAND rm oracle-artemis-test \ No newline at end of file diff --git a/tests/db-tests/scripts/stop-postgres.sh b/tests/db-tests/scripts/stop-postgres.sh new file mode 100755 index 0000000000..a8871353a1 --- /dev/null +++ b/tests/db-tests/scripts/stop-postgres.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source ./container-define.sh + +$CONTAINER_COMMAND kill postgres-artemis-test +$CONTAINER_COMMAND rm postgres-artemis-test \ No newline at end of file diff --git a/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/DropDBTest.java b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/DropDBTest.java new file mode 100644 index 0000000000..35a056d0bf --- /dev/null +++ b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/DropDBTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.tests.db; + +import java.lang.invoke.MethodHandles; +import java.util.Collection; +import java.util.List; + +import org.apache.activemq.artemis.core.server.ActiveMQServer; +import org.apache.activemq.artemis.tests.db.common.Database; +import org.apache.activemq.artemis.tests.db.common.ParameterDBTestBase; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DropDBTest extends ParameterDBTestBase { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Parameterized.Parameters(name = "db={0}") + public static Collection parameters() { + List dbList = Database.selectedList(); + dbList.remove(Database.DERBY); // no derby on this test + return convertParameters(dbList); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + Assume.assumeTrue(database != Database.DERBY); + dropDatabase(); + } + + @Override + protected final String getJDBCClassName() { + return database.getDriverClass(); + } + + @Test + public void testSimpleDrop() throws Exception { + ActiveMQServer server = createServer(createDefaultConfig(0, true)); + server.start(); + server.stop(); + + int tablesDroppped = dropDatabase(); + if (tablesDroppped < 4) { + logger.warn("At least 4 tables should be removed, while only {} tables were dropped", tablesDroppped); + Assert.fail("Only " + tablesDroppped + " tables were dropped"); + } + } +} \ No newline at end of file diff --git a/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/common/DBTestBase.java b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/common/DBTestBase.java new file mode 100644 index 0000000000..012167b662 --- /dev/null +++ b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/common/DBTestBase.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.tests.db.common; + +import java.lang.invoke.MethodHandles; + +import org.apache.activemq.artemis.utils.RealServerTestBase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DBTestBase extends RealServerTestBase { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); +} diff --git a/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/common/Database.java b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/common/Database.java new file mode 100644 index 0000000000..7f078bb561 --- /dev/null +++ b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/common/Database.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.tests.db.common; + +import java.io.File; +import java.net.URL; +import java.net.URLClassLoader; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.util.ArrayList; +import java.util.List; + +import org.apache.activemq.artemis.api.config.ActiveMQDefaultConfiguration; +import org.apache.activemq.artemis.utils.RandomUtil; + +public enum Database { + MYSQL("mysql"), POSTGRES("postgres"), ORACLE("oracle"), MSSQL("mssql"), DB2("db2"), JOURNAL("journal"), DERBY("derby"); + + private String dbname; + private boolean load; + private String jdbcURI; + private String driverClass; + private Driver driver; + private ClassLoader dbClassLoader; + + Database(String dbname) { + this.dbname = dbname; + load = Boolean.parseBoolean(System.getProperty(dbname + ".load", "false")); + } + + public static ClassLoader defineClassLoader(File location, ClassLoader parentClassLoader) throws Exception { + File[] jars = location.listFiles((dir, name) -> name.toLowerCase().endsWith(".jar")); + + URL[] url = new URL[jars.length]; + + for (int i = 0; i < jars.length; i++) { + url[i] = jars[i].toURI().toURL(); + } + + return new URLClassLoader(url, parentClassLoader); + } + + public Connection getConnection() throws Exception { + switch (this) { + case DERBY: + return DriverManager.getConnection(getJdbcURI()); + case JOURNAL: + return null; + default: + return getDriver().connect(getJdbcURI(), null); + } + } + + // There is one artemis server for each database we provide on the tests + public ClassLoader getDBClassLoader() throws Exception { + if (this != JOURNAL && this != DERBY && dbClassLoader == null) { + String serverLocation = ParameterDBTestBase.getServerLocation(getName()); + File lib = new File(serverLocation + "/lib"); + dbClassLoader = defineClassLoader(lib, getClass().getClassLoader()); + } + return dbClassLoader; + } + + public String getName() { + return dbname; + } + + public String getJdbcURI() { + if (jdbcURI == null) { + switch (this) { + case DERBY: + String derbyData = ParameterDBTestBase.getServerLocation("derby") + "/data/derby/db"; + jdbcURI = "jdbc:derby:" + derbyData + ";create=true"; + break; + case JOURNAL: + jdbcURI = null; + break; + default: + jdbcURI = System.getProperty(dbname + ".uri"); + if (jdbcURI != null) { + jdbcURI = jdbcURI.replaceAll("&", "&"); + } + } + } + return jdbcURI; + } + + public String getDriverClass() { + if (driverClass != null) { + return driverClass; + } + + switch (this) { + case DERBY: + this.driverClass = ActiveMQDefaultConfiguration.getDefaultDriverClassName(); + break; + case JOURNAL: + this.driverClass = null; + break; + default: + driverClass = System.getProperty(dbname + ".class"); + } + + return driverClass; + } + + public Driver getDriver() throws Exception { + if (driver == null) { + String className = getDriverClass(); + ClassLoader loader = getDBClassLoader(); + Class clazz = loader.loadClass(className); + driver = (Driver) clazz.getDeclaredConstructor().newInstance(); + } + return driver; + } + + // this must be called within the test classLoader (Thread Context Class Loader) + public void registerDriver() throws Exception { + if (driver != null) { + DriverManager.registerDriver(driver); + } + } + + public boolean isLoad() { + return load; + } + + @Override + public String toString() { + return dbname; + } + + + // it will return a list of Databases selected to be tested + public static List selectedList() { + ArrayList dbList = new ArrayList<>(); + + for (Database db : Database.values()) { + if (db.isLoad()) { + dbList.add(db); + } + } + + return dbList; + } + + public static Database random() { + List selectedDatabases = selectedList(); + if (selectedDatabases.isEmpty()) { + return null; + } else { + return selectedDatabases.get(RandomUtil.randomInterval(0, selectedDatabases.size())); + } + } + + public static List randomList() { + List selectedDatabases = selectedList(); + ArrayList list = new ArrayList<>(); + Database randomDB = random(); + if (randomDB != null) { + list.add(randomDB); + } + return list; + } + +} diff --git a/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/common/ParameterDBTestBase.java b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/common/ParameterDBTestBase.java new file mode 100644 index 0000000000..fa0263173c --- /dev/null +++ b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/common/ParameterDBTestBase.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.tests.db.common; + +import java.lang.invoke.MethodHandles; +import java.sql.Connection; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import org.apache.activemq.artemis.core.config.Configuration; +import org.apache.activemq.artemis.core.config.storage.DatabaseStorageConfiguration; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@RunWith(Parameterized.class) +public abstract class ParameterDBTestBase extends DBTestBase { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Override + protected void dropDerby() throws Exception { + cleanupData(database.getName()); + } + + @Override + protected final String getTestJDBCConnectionUrl() { + return database.getJdbcURI(); + } + + + @Parameterized.Parameter + public Database database; + + @Override + public void setUp() throws Exception { + super.setUp(); + disableCheckThread(); + if (database == Database.DERBY) { + runAfter(this::shutdownDerby); + } + + registerDB(); + + dropDatabase(); + } + + // Register the database on driver and prepares the classLoader to be used + private void registerDB() throws Exception { + ClassLoader dbClassLoader = database.getDBClassLoader(); + if (dbClassLoader != null) { + ClassLoader tccl = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(dbClassLoader); + final Thread currentThread = Thread.currentThread(); + runAfter((() -> { + currentThread.setContextClassLoader(tccl); + })); + database.registerDriver(); + } + } + + + protected static ArrayList convertParameters(List dbList) { + ArrayList parameters = new ArrayList<>(); + dbList.forEach(s -> { + logger.info("Adding {} to the list for the test", s); + parameters.add(new Object[]{s}); + }); + + return parameters; + } + + protected ClassLoader getDBClassLoader() throws Exception { + return database.getDBClassLoader(); + } + + public Connection getConnection() throws Exception { + return database.getConnection(); + } + + public int dropDatabase() { + switch (database) { + case JOURNAL: + return 0; + case DERBY: + try { + logger.info("Drop derby"); + dropDerby(); + } catch (Exception e) { + logger.debug("Error dropping derby db: {}", e.getMessage()); + } + return 1; + default: + return dropTables("BINDING", "MESSAGE", "LARGE_MESSAGE", "PAGE_STORE", "NODE_MANAGER"); + } + } + + + private int dropTables(String...tables) { + int dropped = 0; + + try (Connection connection = getConnection()) { + ResultSet data = connection.getMetaData().getTables(null, "%", "%", new String[]{"TABLE"}); + while (data.next()) { + String table = data.getString("TABLE_NAME"); + + logger.debug("Checking table {}", table); + + boolean drop = false; + for (String str : tables) { + logger.debug("Checking pattern {}", str); + if (table.toUpperCase(Locale.ROOT).contains(str)) { + logger.debug("Table {} is part of the list as it is matching", table, str); + drop = true; + break; + } + } + + if (drop) { + if (dropTable(connection, table)) { + dropped++; + } + } + } + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return -1; + } + + return dropped; + } + + private boolean dropTable(Connection connection, String table) { + try { + connection.createStatement().execute("DROP TABLE " + table); + logger.debug("Dropped {}", table); + return true; + } catch (Exception e) { + logger.warn("Error dropping {} -> {}", table, e.getMessage()); + return false; + } + } + + + @Override + protected Configuration createDefaultConfig(final int serverID, final boolean netty) throws Exception { + Configuration configuration = super.createDefaultConfig(serverID, netty); + if (database != Database.JOURNAL) { + setDBStoreType(configuration); + } + configuration.getAddressSettings().clear(); + return configuration; + } + + @Override + protected DatabaseStorageConfiguration createDefaultDatabaseStorageConfiguration() { + DatabaseStorageConfiguration dbStorageConfiguration = new DatabaseStorageConfiguration(); + String connectionURI = getTestJDBCConnectionUrl(); + dbStorageConfiguration.setJdbcDriverClassName(database.getDriverClass()); + dbStorageConfiguration.setJdbcConnectionUrl(connectionURI); + dbStorageConfiguration.setBindingsTableName("BINDINGS"); + dbStorageConfiguration.setMessageTableName("MESSAGES"); + dbStorageConfiguration.setLargeMessageTableName("LARGE_MESSAGES"); + dbStorageConfiguration.setPageStoreTableName("PAGE_STORE"); + dbStorageConfiguration.setJdbcPassword(getJDBCPassword()); + dbStorageConfiguration.setJdbcUser(getJDBCUser()); + dbStorageConfiguration.setJdbcLockAcquisitionTimeoutMillis(getJdbcLockAcquisitionTimeoutMillis()); + dbStorageConfiguration.setJdbcLockExpirationMillis(getJdbcLockExpirationMillis()); + dbStorageConfiguration.setJdbcLockRenewPeriodMillis(getJdbcLockRenewPeriodMillis()); + dbStorageConfiguration.setJdbcNetworkTimeout(-1); + dbStorageConfiguration.setJdbcAllowedTimeDiff(250L); + return dbStorageConfiguration; + } + + + +} diff --git a/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/invalid/JdbcStartupInvalidTest.java b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/invalid/JdbcStartupInvalidTest.java new file mode 100644 index 0000000000..52a7670cd3 --- /dev/null +++ b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/invalid/JdbcStartupInvalidTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.tests.db.invalid; + +import java.lang.invoke.MethodHandles; +import java.util.concurrent.TimeUnit; + +import org.apache.activemq.artemis.tests.db.common.DBTestBase; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JdbcStartupInvalidTest extends DBTestBase { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + protected static final String SERVER_NAME = "jdbc-bad-driver"; + + @Test + public void startupBadJdbcConnectionTest() throws Exception { + + Process p = startServer(SERVER_NAME, 0, 0); + try { + p.waitFor(20, TimeUnit.SECONDS); + Assert.assertFalse(p.isAlive()); + } catch (Exception e) { + logger.warn(e.getMessage(), e); + Assert.fail(e.getMessage()); + } + } +} diff --git a/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/GlobalPagingTest.java b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/GlobalPagingTest.java new file mode 100644 index 0000000000..45e523675c --- /dev/null +++ b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/GlobalPagingTest.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.tests.db.paging; + +import javax.jms.BytesMessage; +import javax.jms.Connection; +import javax.jms.ConnectionFactory; +import javax.jms.Message; +import javax.jms.MessageConsumer; +import javax.jms.MessageProducer; +import javax.jms.Session; +import java.lang.invoke.MethodHandles; +import java.util.Collection; + +import org.apache.activemq.artemis.api.core.QueueConfiguration; +import org.apache.activemq.artemis.api.core.RoutingType; +import org.apache.activemq.artemis.core.server.ActiveMQServer; +import org.apache.activemq.artemis.core.server.Queue; +import org.apache.activemq.artemis.core.server.impl.AddressInfo; +import org.apache.activemq.artemis.core.settings.impl.AddressSettings; +import org.apache.activemq.artemis.tests.db.common.Database; +import org.apache.activemq.artemis.tests.db.common.ParameterDBTestBase; +import org.apache.activemq.artemis.tests.util.CFUtil; +import org.apache.activemq.artemis.utils.RandomUtil; +import org.apache.activemq.artemis.utils.Wait; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GlobalPagingTest extends ParameterDBTestBase { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Parameterized.Parameters(name = "db={0}") + public static Collection parameters() { + return convertParameters(Database.randomList()); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + dropDatabase(); + } + + @Override + protected final String getJDBCClassName() { + return database.getDriverClass(); + } + + @Test + public void testMaxMessages() throws Exception { + ActiveMQServer server = createServer(createDefaultConfig(0, true)); + server.getConfiguration().getAddressSettings().clear(); + server.getConfiguration().getAddressSettings().put("#", new AddressSettings().setMaxSizeMessages(10)); + server.getConfiguration().setGlobalMaxMessages(5); + server.start(); + String addressName = "Q" + RandomUtil.randomString(); + server.addAddressInfo(new AddressInfo(addressName).addRoutingType(RoutingType.ANYCAST)); + server.createQueue(new QueueConfiguration(addressName).setRoutingType(RoutingType.ANYCAST).setDurable(true)); + + ConnectionFactory cf = CFUtil.createConnectionFactory("core", "tcp://localhost:61616"); + try (Connection connection = cf.createConnection()) { + Session session = connection.createSession(true, Session.SESSION_TRANSACTED); + MessageProducer producer = session.createProducer(session.createQueue(addressName)); + for (int i = 0; i < 6; i++) { + producer.send(session.createTextMessage()); + } + session.commit(); + + Queue queue = server.locateQueue(addressName); + Wait.assertTrue(queue.getPagingStore()::isPaging, 1000, 100); + + connection.start(); + + MessageConsumer consumer = session.createConsumer(session.createQueue(addressName)); + for (int i = 0; i < 6; i++) { + Message message = consumer.receive(5000); + Assert.assertNotNull(message); + } + session.commit(); + + Wait.assertFalse(queue.getPagingStore()::isPaging, 1000, 100); + } + + } + + @Test + public void testMaxMessagesOpposite() throws Exception { + ActiveMQServer server = createServer(createDefaultConfig(0, true)); + server.getConfiguration().getAddressSettings().clear(); + server.getConfiguration().getAddressSettings().put("#", new AddressSettings().setMaxSizeMessages(5)); + server.getConfiguration().setGlobalMaxMessages(500); + server.start(); + String addressName = "Q" + RandomUtil.randomString(); + server.addAddressInfo(new AddressInfo(addressName).addRoutingType(RoutingType.ANYCAST)); + server.createQueue(new QueueConfiguration(addressName).setRoutingType(RoutingType.ANYCAST).setDurable(true)); + + ConnectionFactory cf = CFUtil.createConnectionFactory("core", "tcp://localhost:61616"); + try (Connection connection = cf.createConnection()) { + Session session = connection.createSession(true, Session.SESSION_TRANSACTED); + MessageProducer producer = session.createProducer(session.createQueue(addressName)); + for (int i = 0; i < 6; i++) { + producer.send(session.createTextMessage()); + } + session.commit(); + + Queue queue = server.locateQueue(addressName); + Wait.assertTrue(queue.getPagingStore()::isPaging, 1000, 100); + + connection.start(); + + MessageConsumer consumer = session.createConsumer(session.createQueue(addressName)); + for (int i = 0; i < 6; i++) { + Message message = consumer.receive(5000); + Assert.assertNotNull(message); + } + session.commit(); + + Wait.assertFalse(queue.getPagingStore()::isPaging, 1000, 100); + } + + } + + @Test + public void testMaxMessagesBytes() throws Exception { + ActiveMQServer server = createServer(createDefaultConfig(0, true)); + server.getConfiguration().getAddressSettings().clear(); + server.getConfiguration().getAddressSettings().put("#", new AddressSettings().setMaxSizeMessages(10000).setMaxSizeBytes(100 * 1024 * 1024)); + server.getConfiguration().setGlobalMaxSize(50 * 1024); + server.start(); + String addressName = "Q" + RandomUtil.randomString(); + server.addAddressInfo(new AddressInfo(addressName).addRoutingType(RoutingType.ANYCAST)); + server.createQueue(new QueueConfiguration(addressName).setRoutingType(RoutingType.ANYCAST).setDurable(true)); + + ConnectionFactory cf = CFUtil.createConnectionFactory("core", "tcp://localhost:61616"); + try (Connection connection = cf.createConnection()) { + Session session = connection.createSession(true, Session.SESSION_TRANSACTED); + MessageProducer producer = session.createProducer(session.createQueue(addressName)); + for (int i = 0; i < 6; i++) { + BytesMessage message = session.createBytesMessage(); + message.writeBytes(new byte[20 * 1024]); + producer.send(message); + } + session.commit(); + + Queue queue = server.locateQueue(addressName); + Wait.assertTrue(queue.getPagingStore()::isPaging, 1000, 100); + + connection.start(); + + MessageConsumer consumer = session.createConsumer(session.createQueue(addressName)); + for (int i = 0; i < 6; i++) { + Message message = consumer.receive(5000); + Assert.assertNotNull(message); + } + session.commit(); + + Wait.assertFalse(queue.getPagingStore()::isPaging, 1000, 100); + } + + } + + @Test + public void testMaxMessagesBytesOpposite() throws Exception { + ActiveMQServer server = createServer(createDefaultConfig(0, true)); + server.getConfiguration().getAddressSettings().clear(); + server.getConfiguration().getAddressSettings().put("#", new AddressSettings().setMaxSizeMessages(10000).setMaxSizeBytes(50 * 1024)); + server.getConfiguration().setGlobalMaxSize(10 * 1024 * 1024); + server.start(); + String addressName = "Q" + RandomUtil.randomString(); + server.addAddressInfo(new AddressInfo(addressName).addRoutingType(RoutingType.ANYCAST)); + server.createQueue(new QueueConfiguration(addressName).setRoutingType(RoutingType.ANYCAST).setDurable(true)); + + ConnectionFactory cf = CFUtil.createConnectionFactory("core", "tcp://localhost:61616"); + try (Connection connection = cf.createConnection()) { + Session session = connection.createSession(true, Session.SESSION_TRANSACTED); + MessageProducer producer = session.createProducer(session.createQueue(addressName)); + for (int i = 0; i < 6; i++) { + BytesMessage message = session.createBytesMessage(); + message.writeBytes(new byte[20 * 1024]); + producer.send(message); + } + session.commit(); + + Queue queue = server.locateQueue(addressName); + Wait.assertTrue(queue.getPagingStore()::isPaging, 1000, 100); + + connection.start(); + + MessageConsumer consumer = session.createConsumer(session.createQueue(addressName)); + for (int i = 0; i < 6; i++) { + Message message = consumer.receive(5000); + Assert.assertNotNull(message); + } + session.commit(); + + Wait.assertFalse(queue.getPagingStore()::isPaging, 1000, 100); + } + + } +} \ No newline at end of file diff --git a/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/NetworkTimeoutCheckTest.java b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/NetworkTimeoutCheckTest.java new file mode 100644 index 0000000000..de2e5ab5aa --- /dev/null +++ b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/NetworkTimeoutCheckTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.tests.db.paging; + +import java.lang.invoke.MethodHandles; +import java.util.Collection; + +import org.apache.activemq.artemis.api.core.QueueConfiguration; +import org.apache.activemq.artemis.api.core.RoutingType; +import org.apache.activemq.artemis.core.config.storage.DatabaseStorageConfiguration; +import org.apache.activemq.artemis.core.paging.impl.Page; +import org.apache.activemq.artemis.core.paging.impl.PagingStoreImpl; +import org.apache.activemq.artemis.core.server.ActiveMQServer; +import org.apache.activemq.artemis.core.server.Queue; +import org.apache.activemq.artemis.core.server.impl.AddressInfo; +import org.apache.activemq.artemis.jdbc.store.file.JDBCSequentialFile; +import org.apache.activemq.artemis.tests.db.common.Database; +import org.apache.activemq.artemis.tests.db.common.ParameterDBTestBase; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class NetworkTimeoutCheckTest extends ParameterDBTestBase { + + private static final int TIMEOUT = 33_333; + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Parameterized.Parameters(name = "db={0}") + public static Collection parameters() { + return convertParameters(Database.selectedList()); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + dropDatabase(); + } + + @Override + protected final String getJDBCClassName() { + return database.getDriverClass(); + } + + // check if the sync timeout is propated to the page file + @Test + public void testDBTimeoutPropagated() throws Exception { + ActiveMQServer server = createServer(createDefaultConfig(0, true)); + server.start(); + + + server.addAddressInfo(new AddressInfo(getName()).addRoutingType(RoutingType.ANYCAST)); + Queue queue = server.createQueue(new QueueConfiguration(getName()).setRoutingType(RoutingType.ANYCAST).setDurable(true)); + queue.getPagingStore().startPaging(); + PagingStoreImpl store = (PagingStoreImpl) queue.getPagingStore(); + Page page = store.getCurrentPage(); + JDBCSequentialFile file = (JDBCSequentialFile) page.getFile(); + Assert.assertEquals(TIMEOUT, file.getNetworkTimeoutMillis()); + + server.stop(); + } + + @Override + protected DatabaseStorageConfiguration createDefaultDatabaseStorageConfiguration() { + DatabaseStorageConfiguration dbStorageConfiguration = super.createDefaultDatabaseStorageConfiguration(); + dbStorageConfiguration.setJdbcNetworkTimeout(TIMEOUT); + return dbStorageConfiguration; + } + +} \ No newline at end of file diff --git a/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/PageSizeTest.java b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/PageSizeTest.java new file mode 100644 index 0000000000..1b7f3525a0 --- /dev/null +++ b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/PageSizeTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.tests.db.paging; + +import javax.jms.BytesMessage; +import javax.jms.Connection; +import javax.jms.ConnectionFactory; +import javax.jms.MessageProducer; +import javax.jms.Session; +import java.lang.invoke.MethodHandles; +import java.sql.ResultSet; +import java.util.Collection; + +import org.apache.activemq.artemis.api.core.QueueConfiguration; +import org.apache.activemq.artemis.api.core.RoutingType; +import org.apache.activemq.artemis.core.config.storage.DatabaseStorageConfiguration; +import org.apache.activemq.artemis.core.server.ActiveMQServer; +import org.apache.activemq.artemis.core.server.Queue; +import org.apache.activemq.artemis.core.server.impl.AddressInfo; +import org.apache.activemq.artemis.core.settings.impl.AddressSettings; +import org.apache.activemq.artemis.tests.db.common.Database; +import org.apache.activemq.artemis.tests.db.common.ParameterDBTestBase; +import org.apache.activemq.artemis.tests.util.CFUtil; +import org.apache.activemq.artemis.utils.RandomUtil; +import org.apache.activemq.artemis.utils.Wait; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PageSizeTest extends ParameterDBTestBase { + + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Parameterized.Parameters(name = "db={0}") + public static Collection parameters() { + return convertParameters(Database.selectedList()); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + dropDatabase(); + } + + @Override + protected final String getJDBCClassName() { + return database.getDriverClass(); + } + + @Test + public void testMaxSizePage() throws Exception { + ActiveMQServer server = createServer(createDefaultConfig(0, true)); + server.getConfiguration().getAddressSettings().clear(); + server.getConfiguration().getAddressSettings().put("#", new AddressSettings().setMaxSizeMessages(1)); + DatabaseStorageConfiguration dbstoreConfig = (DatabaseStorageConfiguration) server.getConfiguration().getStoreConfiguration(); + dbstoreConfig.setMaxPageSizeBytes(30 * 1024); + server.start(); + String addressName = "Q" + RandomUtil.randomString(); + server.addAddressInfo(new AddressInfo(addressName).addRoutingType(RoutingType.ANYCAST)); + server.createQueue(new QueueConfiguration(addressName).setRoutingType(RoutingType.ANYCAST).setDurable(true)); + + ConnectionFactory cf = CFUtil.createConnectionFactory("core", "tcp://localhost:61616"); + try (Connection connection = cf.createConnection()) { + Session session = connection.createSession(true, Session.SESSION_TRANSACTED); + MessageProducer producer = session.createProducer(session.createQueue(addressName)); + for (int i = 0; i < 100; i++) { + BytesMessage message = session.createBytesMessage(); + message.writeBytes(new byte[1024]); + producer.send(message); + } + session.commit(); + + Queue queue = server.locateQueue(addressName); + Wait.assertTrue(queue.getPagingStore()::isPaging, 1000, 100); + long size = getMaxSizeBytesStored(queue); + // organically all the pages should have less than 30K + Assert.assertTrue("size is " + size, size <= dbstoreConfig.getMaxPageSizeBytes()); + + // will send one very large message, but bellow the streaming size. We should have one record > page_size + BytesMessage message = session.createBytesMessage(); + message.writeBytes(new byte[50 * 1024]); + producer.send(message); + session.commit(); + + // Since we sent a large message (I mean, not streaming large, but larger than page-size, + // a page with more than 30K should been created to allow the "large" message to be stored. + size = getMaxSizeBytesStored(queue); + Assert.assertTrue("size is " + size, size >= 50 * 1024); + } + } + + + protected long getMaxSizeBytesStored(Queue queue) throws Exception { + String tableName = queue.getPagingStore().getFolderName(); + + try (java.sql.Connection sqlConn = database.getConnection()) { + String sql = null; + switch (database) { + case MSSQL: + sql = "SELECT MAX(LEN(DATA)) FROM " + tableName; + break; + case ORACLE: + case DB2: + case DERBY: + case MYSQL: + sql = "SELECT MAX(LENGTH(DATA)) FROM " + tableName; + break; + case POSTGRES: + sql = "SELECT MAX(OCTET_LENGTH(lo_get(DATA))) FROM " + tableName; + break; + } + logger.debug("query: {}", sql); + if (sql != null) { + ResultSet resultSet = null; + resultSet = sqlConn.createStatement().executeQuery(sql); + Assert.assertTrue(resultSet.next()); + return resultSet.getLong(1); + } else { + return -1; + } + } + } +} \ No newline at end of file diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingTest.java b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/PagingTest.java similarity index 78% rename from tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingTest.java rename to tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/PagingTest.java index cd706b550d..594920c742 100644 --- a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingTest.java +++ b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/PagingTest.java @@ -14,35 +14,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.activemq.artemis.tests.integration.paging; +package org.apache.activemq.artemis.tests.db.paging; import javax.jms.Connection; import javax.jms.ConnectionFactory; +import javax.jms.JMSException; import javax.jms.MessageConsumer; import javax.jms.MessageProducer; +import javax.jms.QueueBrowser; import javax.jms.Session; +import javax.jms.TextMessage; import javax.transaction.xa.XAResource; import javax.transaction.xa.Xid; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.io.PrintStream; -import java.io.PrintWriter; +import java.lang.invoke.MethodHandles; import java.lang.management.ManagementFactory; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -65,11 +65,9 @@ import org.apache.activemq.artemis.api.core.client.MessageHandler; import org.apache.activemq.artemis.api.core.client.ServerLocator; import org.apache.activemq.artemis.api.core.management.QueueControl; import org.apache.activemq.artemis.api.core.management.ResourceNames; -import org.apache.activemq.artemis.cli.commands.tools.PrintData; import org.apache.activemq.artemis.core.client.impl.ClientConsumerInternal; import org.apache.activemq.artemis.core.config.Configuration; import org.apache.activemq.artemis.core.config.DivertConfiguration; -import org.apache.activemq.artemis.core.config.StoreConfiguration; import org.apache.activemq.artemis.core.config.storage.DatabaseStorageConfiguration; import org.apache.activemq.artemis.core.filter.Filter; import org.apache.activemq.artemis.core.io.IOCallback; @@ -88,8 +86,6 @@ import org.apache.activemq.artemis.core.paging.cursor.PageSubscription; import org.apache.activemq.artemis.core.paging.cursor.impl.PageCursorProviderAccessor; import org.apache.activemq.artemis.core.paging.cursor.impl.PageCursorProviderImpl; import org.apache.activemq.artemis.core.paging.cursor.impl.PagePositionImpl; -import org.apache.activemq.artemis.core.paging.impl.Page; -import org.apache.activemq.artemis.core.paging.impl.PageTransactionInfoImpl; import org.apache.activemq.artemis.core.paging.impl.PagingStoreFactoryDatabase; import org.apache.activemq.artemis.core.paging.impl.PagingStoreFactoryNIO; import org.apache.activemq.artemis.core.paging.impl.PagingStoreImpl; @@ -98,72 +94,57 @@ import org.apache.activemq.artemis.core.persistence.OperationContext; import org.apache.activemq.artemis.core.persistence.StorageManager; import org.apache.activemq.artemis.core.persistence.impl.journal.AckDescribe; import org.apache.activemq.artemis.core.persistence.impl.journal.DescribeJournal; -import org.apache.activemq.artemis.core.persistence.impl.journal.DescribeJournal.ReferenceDescribe; import org.apache.activemq.artemis.core.persistence.impl.journal.JournalRecordIds; import org.apache.activemq.artemis.core.persistence.impl.journal.OperationContextImpl; import org.apache.activemq.artemis.core.server.ActiveMQServer; import org.apache.activemq.artemis.core.server.Queue; import org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl; import org.apache.activemq.artemis.core.server.impl.AddressInfo; -import org.apache.activemq.artemis.core.server.impl.QueueImpl; import org.apache.activemq.artemis.core.settings.impl.AddressFullMessagePolicy; import org.apache.activemq.artemis.core.settings.impl.AddressSettings; -import org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory; import org.apache.activemq.artemis.logs.AssertionLoggerHandler; import org.apache.activemq.artemis.spi.core.security.ActiveMQSecurityManagerImpl; +import org.apache.activemq.artemis.tests.db.common.Database; +import org.apache.activemq.artemis.tests.db.common.ParameterDBTestBase; import org.apache.activemq.artemis.tests.util.ActiveMQTestBase; import org.apache.activemq.artemis.tests.util.CFUtil; -import org.apache.activemq.artemis.tests.util.RandomUtil; -import org.apache.activemq.artemis.tests.util.Wait; import org.apache.activemq.artemis.utils.CompositeAddress; -import org.apache.activemq.artemis.utils.RetryRule; +import org.apache.activemq.artemis.utils.RandomUtil; +import org.apache.activemq.artemis.utils.Wait; import org.apache.activemq.artemis.utils.actors.ArtemisExecutor; import org.junit.After; import org.junit.Assert; -import org.junit.Assume; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.invoke.MethodHandles; -@RunWith(Parameterized.class) -public class PagingTest extends ActiveMQTestBase { +public class PagingTest extends ParameterDBTestBase { + protected static final int RECEIVE_TIMEOUT = 5000; + protected static final int PAGE_MAX = 100 * 1024; + protected static final int PAGE_SIZE = 10 * 1024; + static final int MESSAGE_SIZE = 1024; // 1k + static final int LARGE_MESSAGE_SIZE = 100 * 1024; + static final SimpleString ADDRESS = new SimpleString("TestADDRESS"); private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @Rule - public RetryRule retryMethod = new RetryRule(1); - protected ServerLocator locator; protected ActiveMQServer server; protected ClientSessionFactory sf; - static final int MESSAGE_SIZE = 1024; // 1k - static final int LARGE_MESSAGE_SIZE = 100 * 1024; - - protected static final int RECEIVE_TIMEOUT = 5000; - - protected static final int PAGE_MAX = 100 * 1024; - - protected static final int PAGE_SIZE = 10 * 1024; - - protected final StoreConfiguration.StoreType storeType; - - static final SimpleString ADDRESS = new SimpleString("SimpleAddress"); - private AssertionLoggerHandler loggerHandler; - public PagingTest(StoreConfiguration.StoreType storeType) { - this.storeType = storeType; - } + @Parameterized.Parameters(name = "db={0}") + public static Collection parameters() { + ArrayList databases = new ArrayList<>(); + databases.add(Database.JOURNAL); - @Parameterized.Parameters(name = "storeType={0}") - public static Collection data() { - Object[][] params = new Object[][]{{StoreConfiguration.StoreType.FILE}, {StoreConfiguration.StoreType.DATABASE}}; - return Arrays.asList(params); + // PagingTest is quite expensive. And it's not really needed to run every single database every time + // we will just pick one! + databases.addAll(Database.randomList()); + + // we will run PagingTest in one database and the journal to validate the codebase in both implementations + return convertParameters(databases); } @Before @@ -189,46 +170,23 @@ public class PagingTest extends ActiveMQTestBase { locator = createInVMNonHALocator(); } - @Test - public void testTooLongPageStoreTableNamePrefix() throws Exception { - if (storeType == StoreConfiguration.StoreType.DATABASE) { - final Configuration config = createDefaultInVMConfig(); - final DatabaseStorageConfiguration storageConfiguration = (DatabaseStorageConfiguration) config.getStoreConfiguration(); - //set the page store table to be longer than 10 chars -> the paging manager initialization will fail - storageConfiguration.setPageStoreTableName("PAGE_STORE_"); - - final int PAGE_MAX = 20 * 1024; - - final int PAGE_SIZE = 10 * 1024; - - final ActiveMQServer server = createServer(true, config, PAGE_SIZE, PAGE_MAX); - server.start(); - - //due to a failed initialisation of the paging manager, it must be null - Assert.assertNull(server.getPagingManager()); - - server.stop(); - } - } - @Test public void testPageOnLargeMessageMultipleQueues() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig(); - final int PAGE_MAX = 20 * 1024; + final int PAGE_MAX = 0; final int PAGE_SIZE = 10 * 1024; - ActiveMQServer server = createServer(true, config, PAGE_SIZE, PAGE_MAX, -1, -1); + server = createServer(true, config, PAGE_SIZE, PAGE_MAX, -1, -1); server.start(); locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); ClientSessionFactory sf = addSessionFactory(createSessionFactory(locator)); - ClientSession session = sf.createSession(null, null, false, true, true, false, 0); + ClientSession session = sf.createSession(null, null, false, false, false, false, 0); session.createQueue(new QueueConfiguration(ADDRESS.concat("-0")).setAddress(ADDRESS)); session.createQueue(new QueueConfiguration(ADDRESS.concat("-1")).setAddress(ADDRESS)); @@ -237,7 +195,7 @@ public class PagingTest extends ActiveMQTestBase { ClientMessage message = null; - for (int i = 0; i < 201; i++) { + for (int i = 0; i < 50; i++) { message = session.createMessage(true); message.getBodyBuffer().writerIndex(0); @@ -246,6 +204,7 @@ public class PagingTest extends ActiveMQTestBase { producer.send(message); } + session.commit(); session.close(); @@ -256,7 +215,7 @@ public class PagingTest extends ActiveMQTestBase { session.start(); - for (int i = 0; i < 201; i++) { + for (int i = 0; i < 10; i++) { ClientMessage message2 = consumer.receive(10000); Assert.assertNotNull("message was null, ad= " + ad, message2); @@ -268,8 +227,8 @@ public class PagingTest extends ActiveMQTestBase { session.commit(); } else { session.rollback(); - for (int i = 0; i < 100; i++) { - ClientMessage message2 = consumer.receive(10000); + for (int i = 0; i < 25; i++) { + ClientMessage message2 = consumer.receive(1000); Assert.assertNotNull(message2); @@ -292,13 +251,16 @@ public class PagingTest extends ActiveMQTestBase { try (AssertionLoggerHandler loggerHandler = new AssertionLoggerHandler()) { Configuration config = createDefaultInVMConfig(); - final int PAGE_MAX = 20 * 1024; + final int PAGE_MAX = 1024; final int PAGE_SIZE = 10 * 1024; - ActiveMQServer server = createServer(true, config, PAGE_SIZE, PAGE_MAX); + server = createServer(true, config, PAGE_SIZE, PAGE_MAX); server.start(); + server.getAddressSettingsRepository().clear(); + server.getAddressSettingsRepository().addMatch("#", new AddressSettings().setMaxReadPageBytes(-1).setMaxSizeMessages(1)); + final int numberOfBytes = 1024; locator.setBlockOnNonDurableSend(false).setBlockOnDurableSend(false).setBlockOnAcknowledge(false); @@ -319,7 +281,7 @@ public class PagingTest extends ActiveMQTestBase { ClientMessage message = null; - for (int i = 0; i < 201; i++) { + for (int i = 0; i < 20; i++) { message = session.createMessage(true); message.getBodyBuffer().writerIndex(0); @@ -331,14 +293,14 @@ public class PagingTest extends ActiveMQTestBase { } producer.send(message); - session.commit(); } + session.commit(); ClientConsumer consumer = session.createConsumer(ADDRESS.concat("-0")); session.start(); - for (int i = 0; i < 201; i++) { + for (int i = 0; i < 20; i++) { ClientMessage message2 = consumer.receive(10000); Assert.assertNotNull(message2); @@ -347,8 +309,8 @@ public class PagingTest extends ActiveMQTestBase { Assert.assertNotNull(message2); - session.commit(); } + session.commit(); consumer.close(); @@ -373,7 +335,6 @@ public class PagingTest extends ActiveMQTestBase { } } - @Test public void testSimpleCursorIterator() throws Exception { Configuration config = createDefaultInVMConfig(); @@ -382,7 +343,7 @@ public class PagingTest extends ActiveMQTestBase { final int PAGE_SIZE = 10 * 1024; - ActiveMQServer server = createServer(true, config, PAGE_SIZE, PAGE_MAX); + server = createServer(true, config, PAGE_SIZE, PAGE_MAX); server.start(); final int numberOfBytes = 124; @@ -434,15 +395,14 @@ public class PagingTest extends ActiveMQTestBase { for (int i = 0; i < NUMBER_OF_MESSAGES; i++) { Assert.assertTrue(iterator.hasNext()); PagedMessage messageReceived = iterator.next().getPagedMessage(); - System.out.println("Page " + messageReceived.getPageNumber() + " , message = " + messageReceived.getMessageNumber()); + logger.debug("Page {} , message = {}", messageReceived.getPageNumber(), messageReceived.getMessageNumber()); Assert.assertNotNull(messageReceived); - Assert.assertEquals(i, (int)messageReceived.getMessage().getIntProperty("i")); + Assert.assertEquals(i, (int) messageReceived.getMessage().getIntProperty("i")); } Assert.assertFalse(iterator.hasNext()); } - @Test public void testSimpleCursorIteratorLargeMessage() throws Exception { Configuration config = createDefaultInVMConfig(); @@ -451,7 +411,7 @@ public class PagingTest extends ActiveMQTestBase { final int PAGE_SIZE = 10 * 1024; - ActiveMQServer server = createServer(true, config, PAGE_SIZE, PAGE_MAX); + server = createServer(true, config, PAGE_SIZE, PAGE_MAX); server.start(); final int numberOfBytes = 200 * 1024; @@ -500,7 +460,7 @@ public class PagingTest extends ActiveMQTestBase { PagedMessage messageReceived = iterator.next().getPagedMessage(); System.out.println("Page " + messageReceived.getPageNumber() + " , message = " + messageReceived.getMessageNumber()); Assert.assertNotNull(messageReceived); - Assert.assertEquals(i, (int)messageReceived.getMessage().getIntProperty("i")); + Assert.assertEquals(i, (int) messageReceived.getMessage().getIntProperty("i")); } Assert.assertFalse(iterator.hasNext()); @@ -508,7 +468,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testPageCleanup() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -516,7 +475,7 @@ public class PagingTest extends ActiveMQTestBase { server.start(); - final int numberOfMessages = 5000; + final int numberOfMessages = 100; locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); @@ -546,7 +505,7 @@ public class PagingTest extends ActiveMQTestBase { bodyLocal.writeBytes(body); producer.send(message); - if (i % 1000 == 0) { + if (i % 10 == 0) { session.commit(); } } @@ -572,7 +531,7 @@ public class PagingTest extends ActiveMQTestBase { bodyLocal.writeBytes(body); producer.send(message); - if (i % 1000 == 0) { + if (i % 10 == 0) { session.commit(); } } @@ -597,7 +556,7 @@ public class PagingTest extends ActiveMQTestBase { msg = consumer.receive(1000); assertNotNull(msg); msg.acknowledge(); - if (i % 500 == 0) { + if (i % 50 == 0) { session.commit(); } } @@ -616,10 +575,8 @@ public class PagingTest extends ActiveMQTestBase { server.stop(); } - @Test public void testPageReload() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultConfig(true).setJournalSyncNonTransactional(false); @@ -681,7 +638,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testQueueRemoveAll() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -689,7 +645,7 @@ public class PagingTest extends ActiveMQTestBase { server.start(); - final int numberOfMessages = 5000; + final int numberOfMessages = 500; locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); @@ -719,7 +675,7 @@ public class PagingTest extends ActiveMQTestBase { bodyLocal.writeBytes(body); producer.send(message); - if (i % 1000 == 0) { + if (i % 100 == 0) { session.commit(); } } @@ -745,7 +701,7 @@ public class PagingTest extends ActiveMQTestBase { bodyLocal.writeBytes(body); producer.send(message); - if (i % 1000 == 0) { + if (i % 100 == 0) { session.commit(); } } @@ -757,168 +713,14 @@ public class PagingTest extends ActiveMQTestBase { Wait.assertEquals(numberOfMessages * 2, queue::getMessageCount); - QueueControl queueControl = (QueueControl) this.server.getManagementService().getResource(ResourceNames.QUEUE + PagingSendTest.ADDRESS); + QueueControl queueControl = (QueueControl) this.server.getManagementService().getResource(ResourceNames.QUEUE + ADDRESS); int removedMessages = queueControl.removeAllMessages(); Assert.assertEquals(numberOfMessages * 2, removedMessages); } - - @Test - public void testPageCleanupWithInvalidDataTruncated() throws Exception { - testPageCleanupWithInvalidData(true); - } - - @Test - public void testPageCleanupWithInvalidData() throws Exception { - testPageCleanupWithInvalidData(false); - } - - public void testPageCleanupWithInvalidData(boolean truncated) throws Exception { - Assume.assumeTrue(storeType != StoreConfiguration.StoreType.DATABASE); - clearDataRecreateServerDirs(); - - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); - - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); - - server.start(); - - final int numberOfMessages = 100; - - locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); - - sf = createSessionFactory(locator); - - ClientSession session = sf.createSession(false, true, true); - - session.createQueue(new QueueConfiguration(PagingTest.ADDRESS)); - - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); - - ClientMessage message = null; - - byte[] body = new byte[10]; - - ByteBuffer bb = ByteBuffer.wrap(body); - - for (int j = 1; j <= 10; j++) { - bb.put(getSamplebyte(j)); - } - - Queue queue = server.locateQueue(ADDRESS); - queue.getPagingStore().startPaging(); - - queue.getPagingStore().forceAnotherPage(); // forcing an empty file, just to make it more challenging - - int page = 1; - for (int i = 0; i < numberOfMessages; i++) { - if (i % 10 == 0 && i > 0) { - queue.getPagingStore().forceAnotherPage(); - page++; - } - message = session.createMessage(true); - - ActiveMQBuffer bodyLocal = message.getBodyBuffer(); - - bodyLocal.writeBytes(body); - - message.putIntProperty("i", i); - message.putIntProperty("page", page); - - producer.send(message); - } - - queue.getPagingStore().getCursorProvider().disableCleanup(); - - - ClientConsumer consumer = session.createConsumer(ADDRESS); - session.start(); - - for (int i = 0; i < 11; i++) { - ClientMessage msgRec = consumer.receive(1000); - Assert.assertNotNull(msgRec); - msgRec.acknowledge(); - } - session.commit(); - - consumer.close(); - - consumer = session.createConsumer(ADDRESS, SimpleString.toSimpleString("i=29")); - - message = consumer.receive(5000); - Assert.assertNotNull(message); - message.acknowledge(); - session.commit(); - - File folder = queue.getPagingStore().getFolder(); - - // We will truncate two files - for (int f = 2; f <= 3; f++) { - String fileName = ((PagingStoreImpl)queue.getPagingStore()).createFileName(f); - File file = new File(folder, fileName); - file.delete(); - file.createNewFile(); - if (!truncated) { - FileOutputStream fileOutputStream = new FileOutputStream(file); - fileOutputStream.write(new byte[10]); - fileOutputStream.close(); - } - } - sf.close(); - - server.getStorageManager().getMessageJournal().scheduleCompactAndBlock(5000); - - Page page4 = queue.getPagingStore().newPageObject(4); - page4.open(true); - org.apache.activemq.artemis.utils.collections.LinkedList messagesRead = page4.read(server.getStorageManager()); - Assert.assertEquals(10, messagesRead.size()); - page4.close(false); - page4.delete(null); - page4.open(true); - for (int i = 0; i < 9; i++) { - page4.write(messagesRead.get(i)); // this will make message 29 disappear - } - page4.close(false); - - server.stop(); - - server.start(); - - queue = server.locateQueue(ADDRESS); - Assert.assertTrue(queue.getPagingStore().isPaging()); - - queue.getPageSubscription().enableAutoCleanup(); // this should been true already as the server was restarted, just braces and belts - - sf = createSessionFactory(locator); - session = sf.createSession(false, true, true); - - logger.info("*******************************************************************************************************************************"); - logger.info("Creating consumer"); - - consumer = session.createConsumer(ADDRESS); - session.start(); - - for (int i = 20; i < numberOfMessages; i++) { // I made one message disappear on page 4 - if (i != 29) { // I made message 29 disappear - ClientMessage msgClient = consumer.receive(1000); - Assert.assertNotNull(msgClient); - Assert.assertEquals(i, msgClient.getIntProperty("i").intValue()); - msgClient.acknowledge(); - } - } - ClientMessage msgClient = consumer.receiveImmediate(); - Assert.assertNull(msgClient); - session.commit(); - - Wait.assertFalse(queue.getPagingStore()::isPaging, 5000, 100); - } - - - @Test public void testPageReadOneMessage() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -991,10 +793,8 @@ public class PagingTest extends ActiveMQTestBase { Wait.assertFalse(queue.getPagingStore()::isPaging, 5000, 100); } - @Test public void testCleanupMiddlePageSingleQueue() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -1005,7 +805,6 @@ public class PagingTest extends ActiveMQTestBase { server.start(); - server.addAddressInfo(new AddressInfo(address).addRoutingType(RoutingType.MULTICAST)); server.createQueue(new QueueConfiguration(address + "_1").setAddress(address).setRoutingType(RoutingType.MULTICAST).setDurable(true).setFilterString("page<>5")); server.createQueue(new QueueConfiguration(address + "_2").setAddress(address).setRoutingType(RoutingType.MULTICAST).setDurable(true).setFilterString("page=5")); @@ -1081,24 +880,10 @@ public class PagingTest extends ActiveMQTestBase { SequentialFileFactory factory = PagingStoreTestAccessor.getFileFactory(queue.getPagingStore()); Wait.assertEquals(9, () -> factory.listFiles("page").size(), 5000, 100); - - if (storeType != StoreConfiguration.StoreType.DATABASE) { - - ByteArrayOutputStream byteArraystream = new ByteArrayOutputStream(); - final String utf8 = StandardCharsets.UTF_8.name(); - PrintStream printDataStream = new PrintStream(byteArraystream, true, utf8); - - PrintData.printData(server.getConfiguration().getBindingsLocation(), server.getConfiguration().getJournalLocation(), server.getConfiguration().getPagingLocation(), printDataStream, false, false, true, true, -1); - - String data = byteArraystream.toString(utf8); - Assert.assertEquals("PrintData is recreating empty files", 9, factory.listFiles("page").size()); - Assert.assertTrue(data.contains(lastPageConstant)); - } } session.close(); - server.stop(); server.start(); @@ -1137,7 +922,6 @@ public class PagingTest extends ActiveMQTestBase { } - // Send messages in page, consume them // send again... // consume again @@ -1146,7 +930,6 @@ public class PagingTest extends ActiveMQTestBase { // easier to debug an issue during one development... I decided to then keep the simpler test @Test public void testSimpleResume() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -1156,11 +939,10 @@ public class PagingTest extends ActiveMQTestBase { server.start(); - server.addAddressInfo(new AddressInfo(address).addRoutingType(RoutingType.ANYCAST)); server.createQueue(new QueueConfiguration(address).setAddress(address).setRoutingType(RoutingType.ANYCAST).setDurable(true)); - final int numberOfMessages = 100; + final int numberOfMessages = 20; locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); @@ -1182,7 +964,6 @@ public class PagingTest extends ActiveMQTestBase { Queue queue = server.locateQueue(address); - for (int repeat = 0; repeat < 5; repeat++) { queue.getPagingStore().startPaging(); int page = 1; @@ -1223,22 +1004,18 @@ public class PagingTest extends ActiveMQTestBase { Wait.assertFalse(queue.getPagingStore()::isPaging, 5000, 100); } - - - server.stop(); } @Test public void testQueueRetryMessages() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); + server = createServer(true, config, PagingTest.PAGE_SIZE, 100); server.start(); - final int numberOfMessages = 500; + final int numberOfMessages = 50; locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); @@ -1275,10 +1052,6 @@ public class PagingTest extends ActiveMQTestBase { message.putStringProperty(Message.HDR_ORIGINAL_ADDRESS, PagingTest.ADDRESS + "Original"); message.putStringProperty(Message.HDR_ORIGINAL_QUEUE, PagingTest.ADDRESS + "QueueOriginal"); producer.send(message); - - if (i % 1000 == 0) { - session.commit(); - } } session.commit(); producer.close(); @@ -1308,10 +1081,6 @@ public class PagingTest extends ActiveMQTestBase { message.putStringProperty(Message.HDR_ORIGINAL_ADDRESS, PagingTest.ADDRESS + "Original"); message.putStringProperty(Message.HDR_ORIGINAL_QUEUE, PagingTest.ADDRESS + "QueueOriginal"); producer.send(message); - - if (i % 1000 == 0) { - session.commit(); - } } session.commit(); producer.close(); @@ -1323,95 +1092,32 @@ public class PagingTest extends ActiveMQTestBase { Wait.assertEquals(numberOfMessages * 4, queue::getMessageCount); Wait.assertEquals(0, originalQueue::getMessageCount); - QueueControl queueControl = (QueueControl) this.server.getManagementService().getResource(ResourceNames.QUEUE + PagingSendTest.ADDRESS + "Queue"); - QueueControl originalQueueControl = (QueueControl) this.server.getManagementService().getResource(ResourceNames.QUEUE + PagingSendTest.ADDRESS + "QueueOriginal"); + QueueControl queueControl = (QueueControl) this.server.getManagementService().getResource(ResourceNames.QUEUE + ADDRESS + "Queue"); + QueueControl originalQueueControl = (QueueControl) this.server.getManagementService().getResource(ResourceNames.QUEUE + ADDRESS + "QueueOriginal"); queueControl.retryMessages(); Wait.assertEquals(numberOfMessages * 2, queue::getMessageCount, 5000); Wait.assertEquals(numberOfMessages * 2, originalQueue::getMessageCount, 5000); } - @Test - public void testEmptyAddress() throws Exception { - if (storeType == StoreConfiguration.StoreType.FILE) { - clearDataRecreateServerDirs(); - - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); - - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); - - server.start(); - - final int numberOfMessages = 5000; - - locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); - - sf = createSessionFactory(locator); - - ClientSession session = sf.createSession(false, false, false); - - session.createQueue(new QueueConfiguration(PagingTest.ADDRESS).setRoutingType(RoutingType.ANYCAST)); - - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); - - byte[] body = new byte[MESSAGE_SIZE]; - - ByteBuffer bb = ByteBuffer.wrap(body); - - for (int j = 1; j <= MESSAGE_SIZE; j++) { - bb.put(getSamplebyte(j)); - } - - for (int i = 0; i < numberOfMessages; i++) { - ClientMessage message = session.createMessage(true); - - message.getBodyBuffer().writeBytes(body); - - producer.send(message); - if (i % 1000 == 0) { - session.commit(); - } - } - session.commit(); - producer.close(); - session.close(); - - String addressTxt = server.getPagingManager().getPageStore(PagingTest.ADDRESS).getFolder().getAbsolutePath() + File.separator + PagingStoreFactoryNIO.ADDRESS_FILE; - - server.stop(); - - // delete contents of address.txt - new PrintWriter(addressTxt).close(); - - final AtomicBoolean activationFailures = new AtomicBoolean(); - - server.registerActivationFailureListener(exception -> activationFailures.set(true)); - - server.start(); - - server.stop(); - - assertFalse(activationFailures.get()); - } - } - @Test public void testFqqn() throws Exception { final SimpleString queue = RandomUtil.randomSimpleString(); SimpleString fqqn = CompositeAddress.toFullyQualified(ADDRESS, queue); boolean persistentMessages = true; - clearDataRecreateServerDirs(); - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX, -1, -1); + server = createServer(true, config, PagingTest.PAGE_SIZE, 100, -1, -1); server.start(); - final int numberOfMessages = 1000; + server.getAddressSettingsRepository().clear(); + server.getAddressSettingsRepository().addMatch("#", new AddressSettings().setMaxReadPageBytes(-1).setMaxSizeMessages(0).setPageSizeBytes(PagingTest.PAGE_SIZE)); - locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); + final int numberOfMessages = 10; + + locator = createInVMNonHALocator().setBlockOnNonDurableSend(false).setBlockOnDurableSend(false).setBlockOnAcknowledge(false); sf = createSessionFactory(locator); @@ -1435,9 +1141,6 @@ public class PagingTest extends ActiveMQTestBase { message.putIntProperty(new SimpleString("id"), i); producer.send(message); - if (i % 1000 == 0) { - session.commit(); - } } session.commit(); @@ -1455,114 +1158,17 @@ public class PagingTest extends ActiveMQTestBase { message.acknowledge(); assertEquals(i, message.getIntProperty("id").intValue()); - if (i % 1000 == 0) { - session.commit(); - } } session.commit(); - Wait.assertFalse(server.getPagingManager().getPageStore(ADDRESS)::isPaging, 5000, 100); - server.getPagingManager().deletePageStore(fqqn); assertFalse(Arrays.asList(server.getPagingManager().getStoreNames()).contains(ADDRESS)); } - @Test - public void testPurge() throws Exception { - clearDataRecreateServerDirs(); - - Configuration config = createDefaultNettyConfig().setJournalSyncNonTransactional(false); - - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); - - server.start(); - - SimpleString queue = new SimpleString("testPurge:" + RandomUtil.randomString()); - server.addAddressInfo(new AddressInfo(queue, RoutingType.ANYCAST)); - QueueImpl purgeQueue = (QueueImpl) server.createQueue(new QueueConfiguration(queue).setRoutingType(RoutingType.ANYCAST).setMaxConsumers(1).setPurgeOnNoConsumers(true).setAutoCreateAddress(false)); - - ActiveMQConnectionFactory cf = new ActiveMQConnectionFactory(); - Connection connection = cf.createConnection(); - Session session = connection.createSession(true, Session.SESSION_TRANSACTED); - javax.jms.Queue jmsQueue = session.createQueue(queue.toString()); - - MessageProducer producer = session.createProducer(jmsQueue); - - for (int i = 0; i < 100; i++) { - producer.send(session.createTextMessage("hello" + i)); - } - session.commit(); - - Wait.assertEquals(0, purgeQueue::getMessageCount); - - Wait.assertEquals(0, purgeQueue.getPageSubscription().getPagingStore()::getAddressSize); - - MessageConsumer consumer = session.createConsumer(jmsQueue); - - for (int i = 0; i < 100; i++) { - producer.send(session.createTextMessage("hello" + i)); - if (i == 10) { - purgeQueue.getPageSubscription().getPagingStore().startPaging(); - } - } - session.commit(); - - consumer.close(); - - Wait.assertEquals(0, purgeQueue::getMessageCount); - - Wait.assertFalse(purgeQueue.getPageSubscription()::isPaging); - - Wait.assertEquals(0, purgeQueue.getPageSubscription().getPagingStore()::getAddressSize); - consumer = session.createConsumer(jmsQueue); - - for (int i = 0; i < 100; i++) { - purgeQueue.getPageSubscription().getPagingStore().startPaging(); - Assert.assertTrue(purgeQueue.getPageSubscription().isPaging()); - producer.send(session.createTextMessage("hello" + i)); - if (i % 2 == 0) { - session.commit(); - } - } - - session.commit(); - - Wait.assertTrue(purgeQueue.getPageSubscription()::isPaging); - - connection.start(); - - server.getStorageManager().getMessageJournal().scheduleCompactAndBlock(50000); - Assert.assertNotNull(consumer.receive(5000)); - session.commit(); - - consumer.close(); - - Wait.assertEquals(0, purgeQueue::getMessageCount); - Wait.assertEquals(0, purgeQueue.getPageSubscription().getPagingStore()::getAddressSize); - Wait.assertFalse(purgeQueue.getPageSubscription()::isPaging, 5000, 100); - - StorageManager sm = server.getStorageManager(); - - for (int i = 0; i < 1000; i++) { - long tx = sm.generateID(); - PageTransactionInfoImpl txinfo = new PageTransactionInfoImpl(tx); - sm.storePageTransaction(tx, txinfo); - sm.commit(tx); - tx = sm.generateID(); - sm.updatePageTransaction(tx, txinfo, 1); - sm.commit(tx); - } - - server.stop(); - server.start(); - Wait.assertEquals(0, ()->server.getPagingManager().getTransactions().size()); - } - // First page is complete but it wasn't deleted @Test public void testFirstPageCompleteNotDeleted() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -1665,7 +1271,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testPreparedACKAndRestart() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -1841,11 +1446,8 @@ public class PagingTest extends ActiveMQTestBase { session.close(); } - - @Test public void testSimplePreparedAck() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -1855,7 +1457,6 @@ public class PagingTest extends ActiveMQTestBase { final int numberOfMessages = 50; - // namespace for the first client (before the server restart) { locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true).setAckBatchSize(0); @@ -1919,7 +1520,6 @@ public class PagingTest extends ActiveMQTestBase { session.prepare(xidConsumeNoCommit); } - server.stop(); server.start(); @@ -1946,112 +1546,6 @@ public class PagingTest extends ActiveMQTestBase { } } - - @Test - public void testPreparedACKRemoveAndRestart() throws Exception { - Assume.assumeTrue(storeType == StoreConfiguration.StoreType.FILE); - - clearDataRecreateServerDirs(); - - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); - - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); - - server.start(); - - final int numberOfMessages = 10; - - locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true).setAckBatchSize(0); - - sf = createSessionFactory(locator); - - ClientSession session = sf.createSession(false, true, true); - - session.createQueue(new QueueConfiguration(PagingTest.ADDRESS)); - - Queue queue = server.locateQueue(PagingTest.ADDRESS); - - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); - - byte[] body = new byte[MESSAGE_SIZE]; - - ByteBuffer bb = ByteBuffer.wrap(body); - - for (int j = 1; j <= MESSAGE_SIZE; j++) { - bb.put(getSamplebyte(j)); - } - - queue.getPageSubscription().getPagingStore().startPaging(); - - forcePage(queue); - - for (int i = 0; i < numberOfMessages; i++) { - ClientMessage message = session.createMessage(true); - - message.putIntProperty("count", i); - - ActiveMQBuffer bodyLocal = message.getBodyBuffer(); - - bodyLocal.writeBytes(body); - - producer.send(message); - - if (i == 4) { - session.commit(); - queue.getPageSubscription().getPagingStore().forceAnotherPage(); - } - } - - session.commit(); - - session.close(); - - session = sf.createSession(true, false, false); - - - ClientConsumer cons = session.createConsumer(ADDRESS); - - session.start(); - - for (int i = 0; i <= 4; i++) { - Xid xidConsumeNoCommit = newXID(); - session.start(xidConsumeNoCommit, XAResource.TMNOFLAGS); - // First message is consumed, prepared, will be rolled back later - ClientMessage firstMessageConsumed = cons.receive(5000); - assertNotNull(firstMessageConsumed); - firstMessageConsumed.acknowledge(); - session.end(xidConsumeNoCommit, XAResource.TMSUCCESS); - session.prepare(xidConsumeNoCommit); - } - - File pagingFolder = queue.getPageSubscription().getPagingStore().getFolder(); - - server.stop(); - - // remove the very first page. a restart should not fail - File fileToRemove = new File(pagingFolder, "000000001.page"); - Assert.assertTrue(fileToRemove.delete()); - - server.start(); - - sf = createSessionFactory(locator); - - session = sf.createSession(false, true, true); - - cons = session.createConsumer(ADDRESS); - - session.start(); - - for (int i = 5; i < numberOfMessages; i++) { - ClientMessage message = cons.receive(1000); - assertNotNull(message); - assertEquals(i, message.getIntProperty("count").intValue()); - message.acknowledge(); - } - assertNull(cons.receiveImmediate()); - session.commit(); - } - /** * @param queue * @throws InterruptedException @@ -2065,22 +1559,18 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testMoveExpire() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalDirectory(getJournalDir()).setJournalSyncNonTransactional(false).setJournalCompactMinFiles(0) // disable compact .setMessageExpiryScanPeriod(10); server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX, -1, -1); - - AddressSettings defaultSetting = new AddressSettings().setPageSizeBytes(PAGE_SIZE).setMaxSizeBytes(PAGE_MAX).setExpiryAddress(new SimpleString("EXP")).setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE).setMaxReadPageBytes(-1).setMaxReadPageMessages(-1); - - server.getAddressSettingsRepository().clear(); - - server.getAddressSettingsRepository().addMatch("#", defaultSetting); - server.start(); - final int numberOfMessages = 5000; + AddressSettings defaultSetting = new AddressSettings().setPageSizeBytes(PAGE_SIZE).setMaxSizeBytes(0).setExpiryAddress(new SimpleString("EXP")).setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE).setMaxReadPageBytes(-1).setMaxReadPageMessages(-1); + server.getAddressSettingsRepository().clear(); + server.getAddressSettingsRepository().addMatch("#", defaultSetting); + + final int numberOfMessages = 100; locator = createInVMNonHALocator().setConsumerWindowSize(10 * 1024 * 1024).setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); @@ -2110,7 +1600,7 @@ public class PagingTest extends ActiveMQTestBase { for (int i = 0; i < numberOfMessages; i++) { ClientMessage message = session.createMessage(true); - if (i < 1000) { + if (i < 50) { message.setExpiration(System.currentTimeMillis() + 100); } @@ -2121,24 +1611,47 @@ public class PagingTest extends ActiveMQTestBase { bodyLocal.writeBytes(body); producer.send(message); - if (i % 1000 == 0) { - session.commit(); - } } session.commit(); producer.close(); - Wait.assertEquals(1000, qEXP::getMessageCount); + try { + Wait.assertEquals(50, qEXP::getMessageCount); + Wait.assertEquals(50, queue1::getMessageCount); + } catch (Throwable e) { + ConnectionFactory factory = CFUtil.createConnectionFactory("core", "tcp://localhost:61616"); + Connection browserc = factory.createConnection(); + browserc.start(); + Session sessionc = browserc.createSession(false, Session.AUTO_ACKNOWLEDGE); + QueueBrowser consumerc = sessionc.createBrowser(sessionc.createQueue(queue1.getName().toString())); + Enumeration enumeration = consumerc.getEnumeration(); + while (enumeration.hasMoreElements()) { + javax.jms.Message message = enumeration.nextElement(); + logger.debug("Received {} from queue1", message.getIntProperty("tst-count")); + } + consumerc.close(); + consumerc = sessionc.createBrowser(sessionc.createQueue("EXP")); + enumeration = consumerc.getEnumeration(); + while (enumeration.hasMoreElements()) { + javax.jms.Message message = enumeration.nextElement(); + logger.debug("Received {} from EAP", message.getIntProperty("tst-count")); + + } + + consumerc.close(); + + throw e; + } session.start(); ClientConsumer consumer = session.createConsumer(ADDRESS); - for (int i = 0; i < numberOfMessages - 1000; i++) { + for (int i = 0; i < 50; i++) { ClientMessage message = consumer.receive(5000); - assertNotNull(message); + assertNotNull("message " + i + " was null", message); message.acknowledge(); - assertTrue(message.getIntProperty("tst-count") >= 1000); + assertTrue(message.getIntProperty("tst-count") >= 50); } session.commit(); @@ -2151,11 +1664,11 @@ public class PagingTest extends ActiveMQTestBase { consumer = session.createConsumer("EXP"); - for (int i = 0; i < 1000; i++) { + for (int i = 0; i < 50; i++) { ClientMessage message = consumer.receive(5000); assertNotNull(message); message.acknowledge(); - assertTrue(message.getIntProperty("tst-count") < 1000); + assertTrue(message.getIntProperty("tst-count") < 50); } assertNull(consumer.receiveImmediate()); @@ -2172,11 +1685,9 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testDeleteQueueRestart() throws Exception { - clearDataRecreateServerDirs(); - Configuration config = createDefaultInVMConfig().setJournalDirectory(getJournalDir()).setJournalSyncNonTransactional(false).setJournalCompactMinFiles(0); // disable compact - ActiveMQServer server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); + server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); server.start(); @@ -2257,7 +1768,7 @@ public class PagingTest extends ActiveMQTestBase { for (RecordInfo info : journalData.getA()) { if (info.getUserRecordType() == JournalRecordIds.ADD_REF) { - DescribeJournal.ReferenceDescribe ref = (ReferenceDescribe) DescribeJournal.newObjectEncoding(info); + DescribeJournal.ReferenceDescribe ref = (DescribeJournal.ReferenceDescribe) DescribeJournal.newObjectEncoding(info); if (ref.refEncoding.queueID == deletedQueueID) { deletedQueueReferences.add(Long.valueOf(info.id)); @@ -2317,7 +1828,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testPreparePersistent() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -2325,7 +1835,7 @@ public class PagingTest extends ActiveMQTestBase { server.start(); - final int numberOfMessages = 500; + final int numberOfMessages = 50; final int numberOfTX = 10; @@ -2363,9 +1873,6 @@ public class PagingTest extends ActiveMQTestBase { message.putIntProperty(new SimpleString("id"), i); producer.send(message); - if (i % 1000 == 0) { - session.commit(); - } } session.commit(); session.close(); @@ -2498,7 +2005,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testSendOverBlockingNoFlowControl() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -2509,7 +2015,7 @@ public class PagingTest extends ActiveMQTestBase { final int biggerMessageSize = 10 * 1024; - final int numberOfMessages = 500; + final int numberOfMessages = 100; locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true).setProducerWindowSize(-1).setMinLargeMessageSize(1024 * 1024); @@ -2568,7 +2074,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testReceiveImmediate() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -2576,7 +2081,7 @@ public class PagingTest extends ActiveMQTestBase { server.start(); - final int numberOfMessages = 1000; + final int numberOfMessages = 100; locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); @@ -2608,7 +2113,7 @@ public class PagingTest extends ActiveMQTestBase { message.putIntProperty(new SimpleString("id"), i); producer.send(message); - if (i % 1000 == 0) { + if (i % 10 == 0) { session.commit(); } } @@ -2668,396 +2173,12 @@ public class PagingTest extends ActiveMQTestBase { while (timeout > System.currentTimeMillis() && queue.getPageSubscription().getPagingStore().isPaging()) { Thread.sleep(100); } - assertFalse(queue.getPageSubscription().getPagingStore().isPaging()); - - } - - @Test - public void testInabilityToCreateDirectoryDuringPaging() throws Exception { - // this test only applies to file-based stores - Assume.assumeTrue(storeType == StoreConfiguration.StoreType.FILE); - - try (AssertionLoggerHandler loggerHandler = new AssertionLoggerHandler()) { - clearDataRecreateServerDirs(); - - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false).setPagingDirectory("/" + UUID.randomUUID().toString()); - - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); - - server.start(); - - final int numberOfMessages = 100; - - locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); - - sf = createSessionFactory(locator); - - ClientSession session = sf.createSession(false, true, true); - - session.createQueue(new QueueConfiguration(PagingTest.ADDRESS)); - - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); - - ClientMessage message = null; - - byte[] body = new byte[MESSAGE_SIZE]; - - ByteBuffer bb = ByteBuffer.wrap(body); - - for (int j = 1; j <= MESSAGE_SIZE; j++) { - bb.put(getSamplebyte(j)); - } - - for (int i = 0; i < numberOfMessages; i++) { - message = session.createMessage(true); - - ActiveMQBuffer bodyLocal = message.getBodyBuffer(); - - bodyLocal.writeBytes(body); - - message.putIntProperty(new SimpleString("id"), i); - - try { - producer.send(message); - } catch (Exception e) { - // ignore - } - } - assertTrue(Wait.waitFor(() -> server.getState() == ActiveMQServer.SERVER_STATE.STOPPED, 5000, 200)); - session.close(); - sf.close(); - locator.close(); - } finally { - if (storeType != StoreConfiguration.StoreType.DATABASE) { - Assert.assertTrue(loggerHandler.findText("AMQ144010")); - } - } - } - - /** - * This test will remove all the page directories during a restart, simulating a crash scenario. The server should still start after this - */ - @Test - public void testDeletePhysicalPages() throws Exception { - clearDataRecreateServerDirs(); - - Configuration config = createDefaultInVMConfig().setPersistDeliveryCountBeforeDelivery(true); - - config.setJournalSyncNonTransactional(false); - - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); - - server.start(); - - final int numberOfMessages = 1000; - - locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); - - sf = createSessionFactory(locator); - - ClientSession session = sf.createSession(false, false, false); - - session.createQueue(new QueueConfiguration(PagingTest.ADDRESS)); - - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); - - ClientMessage message = null; - - byte[] body = new byte[MESSAGE_SIZE]; - - ByteBuffer bb = ByteBuffer.wrap(body); - - for (int j = 1; j <= MESSAGE_SIZE; j++) { - bb.put(getSamplebyte(j)); - } - - for (int i = 0; i < numberOfMessages; i++) { - message = session.createMessage(true); - - ActiveMQBuffer bodyLocal = message.getBodyBuffer(); - - bodyLocal.writeBytes(body); - - message.putIntProperty(new SimpleString("id"), i); - - producer.send(message); - if (i % 1000 == 0) { - session.commit(); - } - } - session.commit(); - session.close(); - - session = null; - - sf.close(); - locator.close(); - - server.stop(); - - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); - server.start(); - - locator = createInVMNonHALocator(); - sf = createSessionFactory(locator); - - Queue queue = server.locateQueue(ADDRESS); - - Wait.assertEquals(numberOfMessages, queue::getMessageCount); - - int msgReceived = 0; - ClientSession sessionConsumer = sf.createSession(false, false, false); - sessionConsumer.start(); - ClientConsumer consumer = sessionConsumer.createConsumer(PagingTest.ADDRESS); - for (int msgCount = 0; msgCount < numberOfMessages; msgCount++) { - logger.debug("Received {}", msgCount); - msgReceived++; - ClientMessage msg = consumer.receive(5000); - if (msg == null) { - logger.debug("It's null. leaving now"); - sessionConsumer.commit(); - fail("Didn't receive a message"); - } - msg.acknowledge(); - - if (msgCount % 5 == 0) { - logger.debug("commit"); - sessionConsumer.commit(); - } - } - - sessionConsumer.commit(); - - sessionConsumer.close(); - - sf.close(); - - locator.close(); - - Wait.assertEquals(0, queue::getMessageCount); - - long timeout = System.currentTimeMillis() + 5000; - while (timeout > System.currentTimeMillis() && queue.getPageSubscription().getPagingStore().isPaging()) { - Thread.sleep(100); - } - assertFalse(queue.getPageSubscription().getPagingStore().isPaging()); - - server.stop(); - - // Deleting the paging data. Simulating a failure - // a dumb user, or anything that will remove the data - deleteDirectory(new File(getPageDir())); - - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); - server.start(); - - locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); - - sf = createSessionFactory(locator); - - queue = server.locateQueue(ADDRESS); - - sf = createSessionFactory(locator); - session = sf.createSession(false, false, false); - - producer = session.createProducer(PagingTest.ADDRESS); - - for (int i = 0; i < numberOfMessages * 2; i++) { - message = session.createMessage(true); - - ActiveMQBuffer bodyLocal = message.getBodyBuffer(); - - bodyLocal.writeBytes(body); - - message.putIntProperty(new SimpleString("theid"), i); - - producer.send(message); - if (i % 1000 == 0) { - session.commit(); - } - } - - session.commit(); - - server.stop(); - - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); - server.start(); - - locator = createInVMNonHALocator(); - sf = createSessionFactory(locator); - - msgReceived = 0; - sessionConsumer = sf.createSession(false, false, false); - sessionConsumer.start(); - consumer = sessionConsumer.createConsumer(PagingTest.ADDRESS); - for (int msgCount = 0; msgCount < numberOfMessages; msgCount++) { - logger.debug("Received {}", msgCount); - msgReceived++; - ClientMessage msg = consumer.receive(5000); - if (msg == null) { - logger.debug("It's null. leaving now"); - sessionConsumer.commit(); - fail("Didn't receive a message"); - } - msg.acknowledge(); - - if (msgCount % 5 == 0) { - logger.debug("commit"); - sessionConsumer.commit(); - } - } - - sessionConsumer.commit(); - - sessionConsumer.close(); - - } - - // 4 messages are send/received, it creates 2 pages, where for second page there is no delete completion record in journal - // server is restarted and 4 messages sent/received again. There should be no lost message. - @Test - public void testRestartWithCompleteAndDeletedPhysicalPage() throws Exception { - clearDataRecreateServerDirs(); - - Configuration config = createDefaultInVMConfig(); - - final AtomicBoolean mainCleanup = new AtomicBoolean(true); - - class InterruptedCursorProvider extends PageCursorProviderImpl { - - InterruptedCursorProvider(PagingStore pagingStore, - StorageManager storageManager) { - super(pagingStore, storageManager); - } - - @Override - protected void cleanup() { - if (mainCleanup.get()) { - super.cleanup(); - } else { - try { - pagingStore.unlock(); - } catch (Throwable ignored) { - } - } - } - } - - server = new ActiveMQServerImpl(config, ManagementFactory.getPlatformMBeanServer(), new ActiveMQSecurityManagerImpl()) { - @Override - protected PagingStoreFactoryNIO getPagingStoreFactory() { - return new PagingStoreFactoryNIO(this.getStorageManager(), this.getConfiguration().getPagingLocation(), this.getConfiguration().getJournalBufferTimeout_NIO(), this.getScheduledPool(), this.getExecutorFactory(), this.getExecutorFactory(), this.getConfiguration().isJournalSyncNonTransactional(), null) { - @Override - public PageCursorProvider newCursorProvider(PagingStore store, - StorageManager storageManager, - AddressSettings addressSettings, - ArtemisExecutor executor) { - return new InterruptedCursorProvider(store, storageManager); - } - }; - } - - }; - - addServer(server); - - AddressSettings defaultSetting = new AddressSettings().setPageSizeBytes(MESSAGE_SIZE). - setMaxSizeBytes(2 * MESSAGE_SIZE).setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE); - - server.getAddressSettingsRepository().addMatch("#", defaultSetting); - - server.start(); - - locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); - - sf = createSessionFactory(locator); - ClientSession session = sf.createSession(true, true, 0); - session.createQueue(new QueueConfiguration(PagingTest.ADDRESS)); - - Queue queue = server.locateQueue(ADDRESS); - - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); - - ClientMessage message; - - for (int i = 0; i < 4; i++) { - message = session.createMessage(true); - - ActiveMQBuffer bodyLocal = message.getBodyBuffer(); - - bodyLocal.writeBytes(new byte[MESSAGE_SIZE]); - - producer.send(message); - session.commit(); - - //last page (#2, whch contains only message #3) is marked as complete - is full - but no delete complete record is added - if (i == 3) { - queue.getPageSubscription().getPagingStore().forceAnotherPage(); - } - - } - - Wait.assertEquals(3, queue.getPageSubscription().getPagingStore()::getCurrentWritingPage); - - ClientConsumer consumer = session.createConsumer(ADDRESS); - session.start(); - - for (int i = 0; i < 4; i++) { - message = consumer.receive(5000); - Assert.assertNotNull("Before restart - message " + i + " is empty.", message); - message.acknowledge(); - } - - server.stop(); - mainCleanup.set(false); - - // Deleting the paging data. Simulating a failure - // a dumb user, or anything that will remove the data - deleteDirectory(new File(getPageDir())); - - logger.trace("Server restart"); - - server.start(); - - locator = createInVMNonHALocator(); - sf = createSessionFactory(locator); - session = sf.createSession(null, null, false, false, true, false, 0); - producer = session.createProducer(PagingTest.ADDRESS); - - for (int i = 0; i < 4; i++) { - message = session.createMessage(true); - - ActiveMQBuffer bodyLocal = message.getBodyBuffer(); - - bodyLocal.writeBytes(new byte[MESSAGE_SIZE]); - - producer.send(message); - } - session.commit(); - - mainCleanup.set(true); - - queue = server.locateQueue(ADDRESS); - queue.getPageSubscription().cleanupEntries(false); - PageCursorProviderAccessor.cleanup(queue.getPageSubscription().getPagingStore().getCursorProvider()); - - consumer = session.createConsumer(ADDRESS); - session.start(); - - for (int i = 0; i < 4; i++) { - message = consumer.receive(5000); - Assert.assertNotNull("After restart - message " + i + " is empty.", message); - message.acknowledge(); - } - - server.stop(); + Wait.assertFalse(() -> queue.getPageSubscription().getPagingStore().isPaging()); } @Test public void testMissingTXEverythingAcked() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -3065,7 +2186,7 @@ public class PagingTest extends ActiveMQTestBase { server.start(); - final int numberOfMessages = 5000; + final int numberOfMessages = 100; final int numberOfTX = 10; @@ -3170,7 +2291,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testMissingTXEverythingAcked2() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -3281,7 +2401,7 @@ public class PagingTest extends ActiveMQTestBase { } session.commit(); - assertNull(cons.receive(500)); + assertNull(cons.receiveImmediate()); } @@ -3298,15 +2418,13 @@ public class PagingTest extends ActiveMQTestBase { public void testTwoQueuesOneNoRouting() throws Exception { boolean persistentMessages = true; - clearDataRecreateServerDirs(); - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX, -1, -1); server.start(); - final int numberOfMessages = 1000; + final int numberOfMessages = 100; locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); @@ -3333,7 +2451,7 @@ public class PagingTest extends ActiveMQTestBase { message.putIntProperty(new SimpleString("id"), i); producer.send(message); - if (i % 1000 == 0) { + if (i % 10 == 0) { session.commit(); } } @@ -3350,7 +2468,7 @@ public class PagingTest extends ActiveMQTestBase { message.acknowledge(); assertEquals(i, message.getIntProperty("id").intValue()); - if (i % 1000 == 0) { + if (i % 10 == 0) { session.commit(); } } @@ -3367,28 +2485,10 @@ public class PagingTest extends ActiveMQTestBase { Wait.assertFalse(server.getPagingManager().getPageStore(ADDRESS)::isPaging, 5000, 100); } - @Test - public void testSendReceivePagingPersistent() throws Exception { - internaltestSendReceivePaging(true); - } - - @Test - public void testSendReceivePagingNonPersistent() throws Exception { - internaltestSendReceivePaging(false); - } - - @Test - public void testWithDiverts() throws Exception { - internalMultiQueuesTest(true); - } - - @Test - public void testWithMultiQueues() throws Exception { - internalMultiQueuesTest(false); - } - public void internalMultiQueuesTest(final boolean divert) throws Exception { - clearDataRecreateServerDirs(); + + ExecutorService executor = Executors.newFixedThreadPool(2); + runAfter(executor::shutdownNow); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -3406,7 +2506,7 @@ public class PagingTest extends ActiveMQTestBase { server.start(); - final int numberOfMessages = 3000; + final int numberOfMessages = 30; final byte[] body = new byte[MESSAGE_SIZE]; @@ -3418,7 +2518,9 @@ public class PagingTest extends ActiveMQTestBase { final AtomicBoolean running = new AtomicBoolean(true); - class TCount extends Thread { + runAfter(() -> running.set(false)); + + class TCount implements Runnable { Queue queue; @@ -3430,7 +2532,8 @@ public class PagingTest extends ActiveMQTestBase { public void run() { try { while (running.get()) { - // this will be overusing what some users do. flush / getCount + // this is just trying to be annoying on the queue. We will keep bugging it and the operation should all be ok. + // this is similar to an user opening many instances of the management console or some monitoring anti-pattern getMessagesAdded(queue); getMessageCount(queue); Thread.sleep(10); @@ -3441,9 +2544,6 @@ public class PagingTest extends ActiveMQTestBase { } } - TCount tcount1 = null; - TCount tcount2 = null; - try { { locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); @@ -3503,12 +2603,8 @@ public class PagingTest extends ActiveMQTestBase { assertNotSame(queue1, queue2); - tcount1 = new TCount(queue1); - - tcount2 = new TCount(queue2); - - tcount1.start(); - tcount2.start(); + executor.execute(new TCount(queue1)); + executor.execute(new TCount(queue2)); locator = createInVMNonHALocator(); final ClientSessionFactory sf2 = createSessionFactory(locator); @@ -3594,21 +2690,11 @@ public class PagingTest extends ActiveMQTestBase { } } - Wait.assertEquals(0, ()->server.getPagingManager().getTransactions().size()); + Wait.assertEquals(0, () -> server.getPagingManager().getTransactions().size()); } finally { running.set(false); - if (tcount1 != null) { - tcount1.interrupt(); - tcount1.join(); - } - - if (tcount2 != null) { - tcount2.interrupt(); - tcount2.join(); - } - try { server.stop(); } catch (Throwable ignored) { @@ -3619,21 +2705,19 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testMultiQueuesNonPersistentAndPersistent() throws Exception { - clearDataRecreateServerDirs(); - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); server.start(); - final int numberOfMessages = 3000; + final int numberOfMessages = 30; - final byte[] body = new byte[MESSAGE_SIZE]; + final byte[] body = new byte[100]; ByteBuffer bb = ByteBuffer.wrap(body); - for (int j = 1; j <= MESSAGE_SIZE; j++) { + for (int j = 1; j <= 100; j++) { bb.put(getSamplebyte(j)); } @@ -3654,7 +2738,7 @@ public class PagingTest extends ActiveMQTestBase { ClientMessage message = null; for (int i = 0; i < numberOfMessages; i++) { - if (i % 500 == 0) { + if (i % 10 == 0) { session.commit(); } message = session.createMessage(true); @@ -3686,89 +2770,61 @@ public class PagingTest extends ActiveMQTestBase { final AtomicInteger errors = new AtomicInteger(0); - Thread t = new Thread() { - @Override - public void run() { - try { - ClientSession session = sf2.createSession(null, null, false, true, true, false, 0); + ClientSession session = sf2.createSession(null, null, false, true, true, false, 0); - ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS + "-1"); + ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS + "-1"); - session.start(); + session.start(); - for (int i = 0; i < numberOfMessages; i++) { - ClientMessage message2 = consumer.receive(PagingTest.RECEIVE_TIMEOUT); + for (int i = 0; i < numberOfMessages; i++) { + ClientMessage message2 = consumer.receive(PagingTest.RECEIVE_TIMEOUT); - Assert.assertNotNull(message2); + Assert.assertNotNull(message2); - Assert.assertEquals(i, message2.getIntProperty("id").intValue()); + Assert.assertEquals(i, message2.getIntProperty("id").intValue()); - message2.acknowledge(); + message2.acknowledge(); - Assert.assertNotNull(message2); - - if (i % 1000 == 0) { - session.commit(); - } - - try { - assertBodiesEqual(body, message2.getBodyBuffer()); - } catch (AssertionError e) { - if (logger.isDebugEnabled()) { - logger.debug("Expected buffer: {}", ActiveMQTestBase.dumpBytesHex(body, 40)); - logger.debug("Arriving buffer: {}", ActiveMQTestBase.dumpBytesHex(message2.getBodyBuffer().toByteBuffer().array(), 40)); - } - throw e; - } - } - - session.commit(); - - consumer.close(); - - session.close(); - } catch (Throwable e) { - e.printStackTrace(); - errors.incrementAndGet(); - } + Assert.assertNotNull(message2); + if (i % 10 == 0) { + session.commit(); } - }; - t.start(); - t.join(); - - assertEquals(0, errors.get()); - - for (int i = 0; i < 20 && server.getPagingManager().getPageStore(ADDRESS).isPaging(); i++) { - // The delete may be asynchronous, giving some time case it eventually happen asynchronously - Thread.sleep(500); + try { + assertBodiesEqual(body, message2.getBodyBuffer()); + } catch (AssertionError e) { + if (logger.isDebugEnabled()) { + logger.debug("Expected buffer: {}", ActiveMQTestBase.dumpBytesHex(body, 40)); + logger.debug("Arriving buffer: {}", ActiveMQTestBase.dumpBytesHex(message2.getBodyBuffer().toByteBuffer().array(), 40)); + } + throw e; + } } - assertFalse(server.getPagingManager().getPageStore(ADDRESS).isPaging()); + session.commit(); - for (int i = 0; i < 20 && server.getPagingManager().getTransactions().size() != 0; i++) { - // The delete may be asynchronous, giving some time case it eventually happen asynchronously - Thread.sleep(500); - } + consumer.close(); - Wait.assertEquals(0, ()->server.getPagingManager().getTransactions().size()); + session.close(); + + Wait.assertFalse(() -> server.getPagingManager().getPageStore(ADDRESS).isPaging()); + + Wait.assertEquals(0, () -> server.getPagingManager().getTransactions().size()); } private void internaltestSendReceivePaging(final boolean persistentMessages) throws Exception { - clearDataRecreateServerDirs(); - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); server.start(); - final int numberOfIntegers = 256; + final int messageSize = 1024; - final int numberOfMessages = 1000; + final int numberOfMessages = 100; locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); sf = createSessionFactory(locator); @@ -3784,13 +2840,7 @@ public class PagingTest extends ActiveMQTestBase { ClientMessage message = null; - byte[] body = new byte[numberOfIntegers * 4]; - - ByteBuffer bb = ByteBuffer.wrap(body); - - for (int j = 1; j <= numberOfIntegers; j++) { - bb.putInt(j); - } + byte[] body = new byte[messageSize]; for (int i = 0; i < numberOfMessages; i++) { message = session.createMessage(persistentMessages); @@ -3835,7 +2885,7 @@ public class PagingTest extends ActiveMQTestBase { Assert.assertNotNull(message2); - if (i % 1000 == 0) { + if (i % 10 == 0) { session.commit(); } @@ -3872,7 +2922,6 @@ public class PagingTest extends ActiveMQTestBase { */ @Test public void testDepageDuringTransaction() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig(); @@ -3984,7 +3033,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testDepageDuringTransaction2() throws Exception { boolean IS_DURABLE_MESSAGE = true; - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig(); @@ -4001,7 +3049,7 @@ public class PagingTest extends ActiveMQTestBase { ClientSession sessionTransacted = sf.createSession(null, null, false, false, false, false, 0); ClientProducer producerTransacted = sessionTransacted.createProducer(PagingTest.ADDRESS); - ClientSession session = sf.createSession(null, null, false, true, true, false, 0); + ClientSession session = sf.createSession(null, null, false, false, true, false, 0); session.createQueue(new QueueConfiguration(PagingTest.ADDRESS)); ClientMessage firstMessage = sessionTransacted.createMessage(IS_DURABLE_MESSAGE); @@ -4028,7 +3076,11 @@ public class PagingTest extends ActiveMQTestBase { numberOfMessages++; producer.send(message); + if (numberOfMessages % 10 == 0) { + session.commit(); + } } + session.commit(); Assert.assertTrue(server.getPagingManager().getPageStore(PagingTest.ADDRESS).isPaging()); @@ -4092,6 +3144,7 @@ public class PagingTest extends ActiveMQTestBase { } + @Test public void testDepageDuringTransaction3() throws Exception { clearDataRecreateServerDirs(); @@ -4118,43 +3171,41 @@ public class PagingTest extends ActiveMQTestBase { sessionNonTX.start(); - for (int i = 0; i < 50; i++) { + final int MAX_TX = 10; + + Assert.assertTrue("MAX_TX needs to be even", MAX_TX % 2 == 0); + + for (int i = 0; i < MAX_TX; i++) { ClientMessage message = sessionNonTX.createMessage(true); message.getBodyBuffer().writeBytes(body); message.putIntProperty(new SimpleString("id"), i); message.putStringProperty(new SimpleString("tst"), new SimpleString("i=" + i)); + message.putStringProperty(new SimpleString("TX"), "true"); producerTransacted.send(message); if (i % 2 == 0) { - for (int j = 0; j < 20; j++) { + logger.debug("Sending"); + for (int j = 0; j < 10; j++) { ClientMessage msgSend = sessionNonTX.createMessage(true); msgSend.putStringProperty(new SimpleString("tst"), new SimpleString("i=" + i + ", j=" + j)); - msgSend.getBodyBuffer().writeBytes(new byte[10 * 1024]); + msgSend.putStringProperty(new SimpleString("TX"), "false"); + msgSend.getBodyBuffer().writeBytes(new byte[20 * 1024]); producerNonTransacted.send(msgSend); } - assertTrue(server.getPagingManager().getPageStore(PagingTest.ADDRESS).isPaging()); + Wait.assertTrue(() -> server.getPagingManager().getPageStore(PagingTest.ADDRESS).isPaging(), 1000); } else { + logger.debug("Receiving"); ClientConsumer consumer = sessionNonTX.createConsumer(PagingTest.ADDRESS); - for (int j = 0; j < 20; j++) { - ClientMessage msgReceived = consumer.receive(10000); - assertNotNull(msgReceived); + for (int j = 0; j < 10; j++) { + ClientMessage msgReceived = consumer.receive(5000); + assertNotNull("i=" + i + ".. j=" + j, msgReceived); msgReceived.acknowledge(); } consumer.close(); } } - ClientConsumer consumerNonTX = sessionNonTX.createConsumer(PagingTest.ADDRESS); - while (true) { - ClientMessage msgReceived = consumerNonTX.receive(1000); - if (msgReceived == null) { - break; - } - msgReceived.acknowledge(); - } - consumerNonTX.close(); - ClientConsumer consumer = sessionNonTX.createConsumer(PagingTest.ADDRESS); Assert.assertNull(consumer.receiveImmediate()); @@ -4163,7 +3214,7 @@ public class PagingTest extends ActiveMQTestBase { sessionTransacted.close(); - for (int i = 0; i < 50; i++) { + for (int i = 0; i < MAX_TX; i++) { ClientMessage message = consumer.receive(PagingTest.RECEIVE_TIMEOUT); Assert.assertNotNull(message); @@ -4185,7 +3236,9 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testDepageDuringTransaction4() throws Exception { - clearDataRecreateServerDirs(); + + ExecutorService executorService = Executors.newFixedThreadPool(1); + runAfter(executorService::shutdownNow); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false).setJournalSyncTransactional(false); @@ -4195,7 +3248,7 @@ public class PagingTest extends ActiveMQTestBase { final AtomicInteger errors = new AtomicInteger(0); - final int numberOfMessages = 10000; + final int numberOfMessages = 1000; locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(false); @@ -4203,49 +3256,43 @@ public class PagingTest extends ActiveMQTestBase { final byte[] body = new byte[MESSAGE_SIZE]; - Thread producerThread = new Thread() { - @Override - public void run() { - ClientSession sessionProducer = null; - try { - sessionProducer = sf.createSession(false, false); - ClientProducer producer = sessionProducer.createProducer(ADDRESS); - - for (int i = 0; i < numberOfMessages; i++) { - ClientMessage msg = sessionProducer.createMessage(true); - msg.getBodyBuffer().writeBytes(body); - msg.putIntProperty("count", i); - producer.send(msg); - - if (i % 100 == 0 && i != 0) { - sessionProducer.commit(); - // Thread.sleep(500); - } - } - - sessionProducer.commit(); - - } catch (Throwable e) { - e.printStackTrace(); // >> junit report - errors.incrementAndGet(); - } finally { - try { - if (sessionProducer != null) { - sessionProducer.close(); - } - } catch (Throwable e) { - e.printStackTrace(); - errors.incrementAndGet(); - } - } - } - }; - - ClientSession session = sf.createSession(true, true, 0); + ClientSession session = sf.createSession(false, false, 0); session.start(); session.createQueue(new QueueConfiguration(PagingTest.ADDRESS)); - producerThread.start(); + executorService.execute(() -> { + ClientSession sessionProducer = null; + try { + sessionProducer = sf.createSession(false, false); + ClientProducer producer = sessionProducer.createProducer(ADDRESS); + + for (int i = 0; i < numberOfMessages; i++) { + ClientMessage msg = sessionProducer.createMessage(true); + msg.getBodyBuffer().writeBytes(body); + msg.putIntProperty("count", i); + producer.send(msg); + + if (i % 100 == 0 && i != 0) { + sessionProducer.commit(); + } + } + + sessionProducer.commit(); + + } catch (Throwable e) { + e.printStackTrace(); // >> junit report + errors.incrementAndGet(); + } finally { + try { + if (sessionProducer != null) { + sessionProducer.close(); + } + } catch (Throwable e) { + e.printStackTrace(); + errors.incrementAndGet(); + } + } + }); ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS); @@ -4254,7 +3301,7 @@ public class PagingTest extends ActiveMQTestBase { assertNotNull(msg); assertEquals(i, msg.getIntProperty("count").intValue()); msg.acknowledge(); - if (i > 0 && i % 10 == 0) { + if (i > 0 && i % 100 == 0) { session.commit(); } } @@ -4262,8 +3309,6 @@ public class PagingTest extends ActiveMQTestBase { session.close(); - producerThread.join(); - locator.close(); sf.close(); @@ -4273,7 +3318,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testOrderingNonTX() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false).setJournalSyncTransactional(false); @@ -4286,7 +3330,7 @@ public class PagingTest extends ActiveMQTestBase { final AtomicInteger errors = new AtomicInteger(0); - final int numberOfMessages = 2000; + final int numberOfMessages = 200; locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); @@ -4310,12 +3354,12 @@ public class PagingTest extends ActiveMQTestBase { msg.putIntProperty("count", i); producer.send(msg); - if (i == 1000) { + if (i == 100) { // The session is not TX, but we do this just to perform a round trip to the server // and make sure there are no pending messages sessionProducer.commit(); - assertTrue(server.getPagingManager().getPageStore(ADDRESS).isPaging()); + Wait.assertTrue(() -> server.getPagingManager().getPageStore(ADDRESS).isPaging()); ready.countDown(); } } @@ -4354,7 +3398,7 @@ public class PagingTest extends ActiveMQTestBase { ClientMessage msg = consumer.receive(5000); assertNotNull(msg); if (i != msg.getIntProperty("count").intValue()) { - logger.debug("Received {} with property = {}", i, msg.getIntProperty("count")); + logger.debug("Received {} with property = {}", i, msg.getIntProperty("count")); logger.debug("###### different"); } // assertEquals(i, msg.getIntProperty("count").intValue()); @@ -4379,7 +3423,6 @@ public class PagingTest extends ActiveMQTestBase { } public void internalTestPageOnScheduling(final boolean restart) throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -4387,7 +3430,7 @@ public class PagingTest extends ActiveMQTestBase { server.start(); - final int numberOfMessages = 1000; + final int numberOfMessages = 100; final int numberOfBytes = 1024; @@ -4476,7 +3519,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testRollbackOnSend() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig(); @@ -4526,7 +3568,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testRollbackOnSendThenSendMore() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig(); @@ -4588,7 +3629,7 @@ public class PagingTest extends ActiveMQTestBase { ClientConsumer consumer = consumerSession.createConsumer(ADDRESS, SimpleString.toSimpleString("id > 0")); for (int i = 0; i < 19; i++) { ClientMessage messageRec = consumer.receive(5000); - System.err.println("msg::" + messageRec); + logger.debug("msg::{}", messageRec); Assert.assertNotNull(messageRec); messageRec.acknowledge(); consumerSession.commit(); @@ -4610,7 +3651,6 @@ public class PagingTest extends ActiveMQTestBase { // The pages are complete, and this is simulating a scenario where the server crashed before deleting the pages. @Test public void testRestartWithComplete() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig(); @@ -4618,8 +3658,7 @@ public class PagingTest extends ActiveMQTestBase { class InterruptedCursorProvider extends PageCursorProviderImpl { - InterruptedCursorProvider(PagingStore pagingStore, - StorageManager storageManager) { + InterruptedCursorProvider(PagingStore pagingStore, StorageManager storageManager) { super(pagingStore, storageManager); } @@ -4654,7 +3693,7 @@ public class PagingTest extends ActiveMQTestBase { addServer(server); - AddressSettings defaultSetting = new AddressSettings().setPageSizeBytes(PagingTest.PAGE_SIZE).setMaxSizeBytes(PagingTest.PAGE_MAX).setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE); + AddressSettings defaultSetting = new AddressSettings().setPageSizeBytes(PagingTest.PAGE_SIZE).setMaxSizeBytes(PagingTest.PAGE_MAX).setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE).setMaxReadPageBytes(-1); server.getAddressSettingsRepository().addMatch("#", defaultSetting); @@ -4691,7 +3730,7 @@ public class PagingTest extends ActiveMQTestBase { } - Wait.assertEquals(20, ()->queue1.getPageSubscription().getPagingStore().getCurrentWritingPage()); + Wait.assertEquals(20, () -> queue1.getPageSubscription().getPagingStore().getCurrentWritingPage()); // This will force a scenario where the pages are cleaned up. When restarting we need to check if the current page is complete // if it is complete we must move to another page avoiding races on cleanup @@ -4758,8 +3797,7 @@ public class PagingTest extends ActiveMQTestBase { } @Test - public void testCommitOnSend() throws Exception { - clearDataRecreateServerDirs(); + public void testPartialConsume() throws Exception { Configuration config = createDefaultInVMConfig(); @@ -4767,77 +3805,7 @@ public class PagingTest extends ActiveMQTestBase { server.start(); - final int numberOfIntegers = 10; - - final int numberOfMessages = 500; - - locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); - - sf = createSessionFactory(locator); - ClientSession session = sf.createSession(null, null, false, false, false, false, 0); - - session.createQueue(new QueueConfiguration(PagingTest.ADDRESS)); - - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); - - ClientMessage message = null; - - for (int i = 0; i < numberOfMessages; i++) { - message = session.createMessage(true); - - ActiveMQBuffer bodyLocal = message.getBodyBuffer(); - - for (int j = 1; j <= numberOfIntegers; j++) { - bodyLocal.writeInt(j); - } - - message.putIntProperty(new SimpleString("id"), i); - - producer.send(message); - } - - session.commit(); - - session.close(); - - locator.close(); - - locator = createInVMNonHALocator(); - - server.stop(); - - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); - - server.start(); - - sf = createSessionFactory(locator); - - session = sf.createSession(null, null, false, false, false, false, 0); - - ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS); - - session.start(); - for (int i = 0; i < numberOfMessages; i++) { - ClientMessage msg = consumer.receive(5000); - Assert.assertNotNull(msg); - msg.acknowledge(); - session.commit(); - } - - session.close(); - } - - @Test - public void testParialConsume() throws Exception { - clearDataRecreateServerDirs(); - - Configuration config = createDefaultInVMConfig(); - - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); - - server.start(); - - final int numberOfMessages = 1000; + final int numberOfMessages = 100; locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); @@ -4883,11 +3851,10 @@ public class PagingTest extends ActiveMQTestBase { ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS); session.start(); - // 347 = I just picked any odd number, not rounded, to make sure it's not at the beginning of any page - for (int i = 0; i < 347; i++) { + for (int i = 0; i < 37; i++) { ClientMessage msg = consumer.receive(5000); - assertEquals(i, msg.getIntProperty("id").intValue()); Assert.assertNotNull(msg); + assertEquals(i, msg.getIntProperty("id").intValue()); msg.acknowledge(); session.commit(); } @@ -4911,10 +3878,10 @@ public class PagingTest extends ActiveMQTestBase { consumer = session.createConsumer(PagingTest.ADDRESS); session.start(); - for (int i = 347; i < numberOfMessages; i++) { + for (int i = 37; i < numberOfMessages; i++) { ClientMessage msg = consumer.receive(5000); - assertEquals(i, msg.getIntProperty("id").intValue()); Assert.assertNotNull(msg); + assertEquals(i, msg.getIntProperty("id").intValue()); msg.acknowledge(); session.commit(); } @@ -4943,12 +3910,11 @@ public class PagingTest extends ActiveMQTestBase { } public void testDropMessages(final boolean persistent) throws Exception { - clearDataRecreateServerDirs(); HashMap settings = new HashMap<>(); - AddressSettings set = new AddressSettings(); - set.setAddressFullMessagePolicy(AddressFullMessagePolicy.DROP); + AddressSettings set = new AddressSettings().setMaxReadPageBytes(-1); + set.setAddressFullMessagePolicy(AddressFullMessagePolicy.DROP).setMaxSizeMessages(5); settings.put(PagingTest.ADDRESS.toString(), set); @@ -4956,12 +3922,12 @@ public class PagingTest extends ActiveMQTestBase { server.start(); - final int numberOfMessages = 1000; + final int numberOfMessages = 10; locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); sf = createSessionFactory(locator); - ClientSession session = sf.createSession(null, null, false, true, true, false, 0); + ClientSession session = sf.createSession(null, null, false, false, true, false, 0); session.createQueue(new QueueConfiguration(PagingTest.ADDRESS)); @@ -4977,6 +3943,7 @@ public class PagingTest extends ActiveMQTestBase { producer.send(message); } + session.commit(); ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS); @@ -4992,7 +3959,7 @@ public class PagingTest extends ActiveMQTestBase { Assert.assertNull(consumer.receiveImmediate()); - Wait.assertEquals(0, ()->server.getPagingManager().getPageStore(PagingTest.ADDRESS).getAddressSize()); + Wait.assertEquals(0, () -> server.getPagingManager().getPageStore(PagingTest.ADDRESS).getAddressSize()); for (int i = 0; i < numberOfMessages; i++) { byte[] body = new byte[2048]; @@ -5002,6 +3969,7 @@ public class PagingTest extends ActiveMQTestBase { producer.send(message); } + session.commit(); for (int i = 0; i < 5; i++) { ClientMessage message2 = consumer.receive(PagingTest.RECEIVE_TIMEOUT); @@ -5048,16 +4016,15 @@ public class PagingTest extends ActiveMQTestBase { session.close(); - Wait.assertEquals(0, ()->server.getPagingManager().getPageStore(PagingTest.ADDRESS).getAddressSize()); + Wait.assertEquals(0, () -> server.getPagingManager().getPageStore(PagingTest.ADDRESS).getAddressSize()); } @Test public void testDropMessagesExpiring() throws Exception { - clearDataRecreateServerDirs(); HashMap settings = new HashMap<>(); - AddressSettings set = new AddressSettings(); + AddressSettings set = new AddressSettings().setMaxReadPageBytes(-1); set.setAddressFullMessagePolicy(AddressFullMessagePolicy.DROP); settings.put(PagingTest.ADDRESS.toString(), set); @@ -5131,7 +4098,7 @@ public class PagingTest extends ActiveMQTestBase { private void internalTestPageMultipleDestinations(final boolean transacted) throws Exception { Configuration config = createDefaultInVMConfig(); - final int NUMBER_OF_BINDINGS = 100; + final int NUMBER_OF_BINDINGS = 10; int NUMBER_OF_MESSAGES = 2; @@ -5229,7 +4196,7 @@ public class PagingTest extends ActiveMQTestBase { PagingStore store = paging.getPageStore(ADDRESS); - store.sync(); + store.addSyncPoint(OperationContextImpl.getContext()); assertTrue(pageUp.await(10, TimeUnit.SECONDS)); @@ -5267,7 +4234,7 @@ public class PagingTest extends ActiveMQTestBase { PagingStore store = paging.getPageStore(ADDRESS); - store.sync(); + store.addSyncPoint(OperationContextImpl.getContext()); assertTrue(pageUp.await(10, TimeUnit.SECONDS)); @@ -5283,11 +4250,13 @@ public class PagingTest extends ActiveMQTestBase { Map addresses = new HashMap<>(); - addresses.put("#", new AddressSettings()); + addresses.put("#", new AddressSettings().setMaxReadPageBytes(-1)); - AddressSettings pagedDestination = new AddressSettings().setPageSizeBytes(1024).setMaxSizeBytes(0); + AddressSettings pagedDestination = new AddressSettings().setPageSizeBytes(1024).setMaxSizeBytes(0).setMaxReadPageBytes(-1); + AddressSettings nonPagedDestination = new AddressSettings().setPageSizeBytes(1024).setMaxSizeBytes(-1).setMaxSizeMessages(-1).setMaxReadPageBytes(-1); addresses.put(PAGED_ADDRESS.toString(), pagedDestination); + addresses.put(NON_PAGED_ADDRESS.toString(), nonPagedDestination); server = createServer(true, configuration, -1, -1, addresses); @@ -5304,7 +4273,7 @@ public class PagingTest extends ActiveMQTestBase { ClientProducer producerPaged = session.createProducer(PAGED_ADDRESS); ClientProducer producerNonPaged = session.createProducer(NON_PAGED_ADDRESS); - int NUMBER_OF_MESSAGES = 100; + int NUMBER_OF_MESSAGES = 10; for (int i = 0; i < NUMBER_OF_MESSAGES; i++) { ClientMessage msg = session.createMessage(true); @@ -5317,7 +4286,7 @@ public class PagingTest extends ActiveMQTestBase { session.close(); Assert.assertTrue(server.getPagingManager().getPageStore(PAGED_ADDRESS).isPaging()); - Assert.assertFalse(server.getPagingManager().getPageStore(NON_PAGED_ADDRESS).isPaging()); + Wait.assertFalse(() -> server.getPagingManager().getPageStore(NON_PAGED_ADDRESS).isPaging()); session = sf.createSession(false, true, false); @@ -5358,7 +4327,6 @@ public class PagingTest extends ActiveMQTestBase { session.close(); } - @Test public void testSimplePaging() throws Exception { SimpleString PAGED_ADDRESS = new SimpleString("paged"); @@ -5367,9 +4335,9 @@ public class PagingTest extends ActiveMQTestBase { Map addresses = new HashMap<>(); - addresses.put("#", new AddressSettings()); + addresses.put("#", new AddressSettings().setMaxReadPageBytes(-1)); - AddressSettings pagedDestination = new AddressSettings().setPageSizeBytes(1024).setMaxSizeBytes(0); + AddressSettings pagedDestination = new AddressSettings().setPageSizeBytes(1024).setMaxSizeBytes(0).setMaxReadPageBytes(-1); addresses.put(PAGED_ADDRESS.toString(), pagedDestination); @@ -5379,20 +4347,14 @@ public class PagingTest extends ActiveMQTestBase { sf = createSessionFactory(locator); - ClientSession session = sf.createSession(false, true, false); + ClientSession session = sf.createSession(false, false, false); session.createQueue(new QueueConfiguration(PAGED_ADDRESS)); Wait.assertTrue(() -> null != server.locateQueue(PAGED_ADDRESS)); - Queue serverQueue = server.locateQueue(PAGED_ADDRESS); - - //serverQueue.getPagingStore().startPaging(); - - //Wait.assertTrue(serverQueue.getPagingStore()::isPaging); - ClientProducer producerPaged = session.createProducer(PAGED_ADDRESS); - int NUMBER_OF_MESSAGES = 100; + int NUMBER_OF_MESSAGES = 10; for (int i = 0; i < NUMBER_OF_MESSAGES; i++) { ClientMessage msg = session.createMessage(true); @@ -5401,11 +4363,10 @@ public class PagingTest extends ActiveMQTestBase { producerPaged.send(msg); } + session.commit(); session.close(); - //System.exit(-1); - Assert.assertTrue(server.getPagingManager().getPageStore(PAGED_ADDRESS).isPaging()); session = sf.createSession(false, true, false); @@ -5420,13 +4381,6 @@ public class PagingTest extends ActiveMQTestBase { Assert.assertNotNull("expected message at " + i, msg); int recI = msg.getIntProperty("i"); Assert.assertEquals(i, recI); - if (recI != i) { - System.err.println("Expected " + i + " and received " + recI); - } - System.out.println("*******************************************************************************************************************************"); - System.out.println("msg " + msg); - System.out.println("*******************************************************************************************************************************"); - msg.acknowledge(); session.commit(); } @@ -5435,221 +4389,10 @@ public class PagingTest extends ActiveMQTestBase { session.close(); } - @Test - public void testPagingDifferentSizes() throws Exception { - SimpleString PAGED_ADDRESS_A = new SimpleString("paged-a"); - SimpleString PAGED_ADDRESS_B = new SimpleString("paged-b"); - - Configuration configuration = createDefaultInVMConfig(); - - Map addresses = new HashMap<>(); - - addresses.put("#", new AddressSettings()); - - AddressSettings pagedDestinationA = new AddressSettings().setPageSizeBytes(1024).setMaxSizeBytes(10 * 1024); - - int NUMBER_MESSAGES_BEFORE_PAGING = 11; - - addresses.put(PAGED_ADDRESS_A.toString(), pagedDestinationA); - - AddressSettings pagedDestinationB = new AddressSettings().setPageSizeBytes(2024).setMaxSizeBytes(20 * 1024); - - addresses.put(PAGED_ADDRESS_B.toString(), pagedDestinationB); - - server = createServer(true, configuration, -1, -1, addresses); - server.start(); - - sf = createSessionFactory(locator); - - ClientSession session = sf.createSession(false, true, false); - - session.createQueue(new QueueConfiguration(PAGED_ADDRESS_A)); - - session.createQueue(new QueueConfiguration(PAGED_ADDRESS_B)); - - ClientProducer producerA = session.createProducer(PAGED_ADDRESS_A); - ClientProducer producerB = session.createProducer(PAGED_ADDRESS_B); - - int NUMBER_OF_MESSAGES = 100; - - for (int i = 0; i < NUMBER_MESSAGES_BEFORE_PAGING; i++) { - ClientMessage msg = session.createMessage(true); - msg.getBodyBuffer().writeBytes(new byte[896]); - - producerA.send(msg); - producerB.send(msg); - } - - session.commit(); // commit was called to clean the buffer only (making sure everything is on the server side) - - Assert.assertTrue(server.getPagingManager().getPageStore(PAGED_ADDRESS_A).isPaging()); - Assert.assertFalse(server.getPagingManager().getPageStore(PAGED_ADDRESS_B).isPaging()); - - for (int i = 0; i < NUMBER_MESSAGES_BEFORE_PAGING; i++) { - ClientMessage msg = session.createMessage(true); - msg.getBodyBuffer().writeBytes(new byte[896]); - - producerA.send(msg); - producerB.send(msg); - } - - session.commit(); // commit was called to clean the buffer only (making sure everything is on the server side) - - Assert.assertTrue(server.getPagingManager().getPageStore(PAGED_ADDRESS_A).isPaging()); - Assert.assertTrue(server.getPagingManager().getPageStore(PAGED_ADDRESS_B).isPaging()); - - for (int i = NUMBER_MESSAGES_BEFORE_PAGING * 2; i < NUMBER_OF_MESSAGES; i++) { - ClientMessage msg = session.createMessage(true); - msg.getBodyBuffer().writeBytes(new byte[896]); - - producerA.send(msg); - producerB.send(msg); - } - - session.close(); - - Assert.assertTrue(server.getPagingManager().getPageStore(PAGED_ADDRESS_A).isPaging()); - Assert.assertTrue(server.getPagingManager().getPageStore(PAGED_ADDRESS_B).isPaging()); - - session = sf.createSession(null, null, false, true, true, false, 0); - - session.start(); - - ClientConsumer consumerA = session.createConsumer(PAGED_ADDRESS_A); - - ClientConsumer consumerB = session.createConsumer(PAGED_ADDRESS_B); - - for (int i = 0; i < NUMBER_OF_MESSAGES; i++) { - ClientMessage msg = consumerA.receive(5000); - Assert.assertNotNull("Couldn't receive a message on consumerA, iteration = " + i, msg); - msg.acknowledge(); - } - - Assert.assertNull(consumerA.receiveImmediate()); - - consumerA.close(); - - Assert.assertTrue(server.getPagingManager().getPageStore(PAGED_ADDRESS_B).isPaging()); - - for (int i = 0; i < NUMBER_OF_MESSAGES; i++) { - ClientMessage msg = consumerB.receive(5000); - Assert.assertNotNull(msg); - msg.acknowledge(); - session.commit(); - } - - Assert.assertNull(consumerB.receiveImmediate()); - - consumerB.close(); - - session.close(); - } - - @Test - public void testPageAndDepageRapidly() throws Exception { - boolean persistentMessages = true; - - clearDataRecreateServerDirs(); - - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false).setJournalFileSize(10 * 1024 * 1024); - - server = createServer(true, config, 100 * 1024, 1024 * 1024 / 2, -1, -1); - - server.start(); - - final int messageSize = 51527; - - final int numberOfMessages = 200; - - locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); - - sf = createSessionFactory(locator); - - ClientSession session = sf.createSession(true, true); - - session.createQueue(new QueueConfiguration(PagingTest.ADDRESS)); - - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); - - final AtomicInteger errors = new AtomicInteger(0); - - Thread consumeThread = new Thread() { - @Override - public void run() { - ClientSession sessionConsumer = null; - try { - sessionConsumer = sf.createSession(false, false); - sessionConsumer.start(); - - ClientConsumer cons = sessionConsumer.createConsumer(ADDRESS); - - for (int i = 0; i < numberOfMessages; i++) { - ClientMessage msg = cons.receive(PagingTest.RECEIVE_TIMEOUT); - assertNotNull(msg); - msg.acknowledge(); - - if (i % 20 == 0) { - sessionConsumer.commit(); - } - } - sessionConsumer.commit(); - } catch (Throwable e) { - e.printStackTrace(); - errors.incrementAndGet(); - } finally { - try { - sessionConsumer.close(); - } catch (ActiveMQException e) { - e.printStackTrace(); - errors.incrementAndGet(); - } - } - - } - }; - - consumeThread.start(); - - ClientMessage message = null; - - byte[] body = new byte[messageSize]; - - for (int i = 0; i < numberOfMessages; i++) { - message = session.createMessage(persistentMessages); - - ActiveMQBuffer bodyLocal = message.getBodyBuffer(); - - bodyLocal.writeBytes(body); - - message.putIntProperty(new SimpleString("id"), i); - - producer.send(message); - - Thread.sleep(50); - } - - consumeThread.join(); - - assertEquals(0, errors.get()); - - long timeout = System.currentTimeMillis() + 5000; - - while (System.currentTimeMillis() < timeout && (server.getPagingManager().getPageStore(ADDRESS).isPaging() || server.getPagingManager().getPageStore(ADDRESS).getNumberOfPages() != 1)) { - Thread.sleep(1); - } - - // It's async, so need to wait a bit for it happening - Wait.assertFalse(server.getPagingManager().getPageStore(ADDRESS)::isPaging); - - Wait.assertEquals(1, ()->server.getPagingManager().getPageStore(ADDRESS).getNumberOfPages()); - } - @Test public void testTwoQueuesDifferentFilters() throws Exception { boolean persistentMessages = true; - clearDataRecreateServerDirs(); - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX, -1, -1); @@ -5730,8 +4473,6 @@ public class PagingTest extends ActiveMQTestBase { public void testTwoQueues() throws Exception { boolean persistentMessages = true; - clearDataRecreateServerDirs(); - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX, -1, -1); @@ -5740,7 +4481,7 @@ public class PagingTest extends ActiveMQTestBase { final int messageSize = 1024; - final int numberOfMessages = 1000; + final int numberOfMessages = 100; try { ServerLocator locator = createInVMNonHALocator().setClientFailureCheckPeriod(120000).setConnectionTTL(5000000).setCallTimeout(120000).setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); @@ -5768,9 +4509,6 @@ public class PagingTest extends ActiveMQTestBase { message.putIntProperty("propTest", i % 2 == 0 ? 1 : 2); producer.send(message); - if (i % 1000 == 0) { - session.commit(); - } } session.commit(); @@ -5797,10 +4535,7 @@ public class PagingTest extends ActiveMQTestBase { PagingStore store = server.getPagingManager().getPageStore(ADDRESS); PageCursorProviderAccessor.cleanup(store.getCursorProvider()); - long timeout = System.currentTimeMillis() + 5000; - while (store.isPaging() && timeout > System.currentTimeMillis()) { - Thread.sleep(100); - } + Wait.waitFor(() -> !store.isPaging(), 5000); PageCursorProviderAccessor.cleanup(store.getCursorProvider()); @@ -5821,8 +4556,6 @@ public class PagingTest extends ActiveMQTestBase { public void testTwoQueuesAndOneInativeQueue() throws Exception { boolean persistentMessages = true; - clearDataRecreateServerDirs(); - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); @@ -5896,8 +4629,6 @@ public class PagingTest extends ActiveMQTestBase { public void testTwoQueuesConsumeOneRestart() throws Exception { boolean persistentMessages = true; - clearDataRecreateServerDirs(); - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX, -1, -1); @@ -5906,7 +4637,7 @@ public class PagingTest extends ActiveMQTestBase { final int messageSize = 1024; - final int numberOfMessages = 1000; + final int numberOfMessages = 100; try { ServerLocator locator = createInVMNonHALocator().setClientFailureCheckPeriod(120000).setConnectionTTL(5000000).setCallTimeout(120000).setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); @@ -5934,9 +4665,6 @@ public class PagingTest extends ActiveMQTestBase { message.putIntProperty("propTest", i % 2 == 0 ? 1 : 2); producer.send(message); - if (i % 1000 == 0) { - session.commit(); - } } session.commit(); @@ -5996,19 +4724,18 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testDLAOnLargeMessageAndPaging() throws Exception { - clearDataRecreateServerDirs(); - Configuration config = createDefaultInVMConfig().setThreadPoolMaxSize(5).setJournalSyncNonTransactional(false); + Configuration config = createDefaultInVMConfig().setThreadPoolMaxSize(10).setJournalSyncNonTransactional(false); Map settings = new HashMap<>(); - AddressSettings dla = new AddressSettings().setMaxDeliveryAttempts(5).setDeadLetterAddress(new SimpleString("DLA")).setRedeliveryDelay(0); + AddressSettings dla = new AddressSettings().setMaxDeliveryAttempts(2).setDeadLetterAddress(new SimpleString("DLA")).setRedeliveryDelay(0).setMaxReadPageBytes(-1); settings.put(ADDRESS.toString(), dla); server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX, -1, -1, settings); server.start(); - final int messageSize = 1024; + final int messageSize = 124; ServerLocator locator = null; ClientSessionFactory sf = null; @@ -6027,7 +4754,6 @@ public class PagingTest extends ActiveMQTestBase { session.createQueue(new QueueConfiguration("DLA")); Queue serverQueue = server.locateQueue(ADDRESS); - Queue serverQueueDLA = server.locateQueue(SimpleString.toSimpleString("DLA")); PagingStore pgStoreAddress = server.getPagingManager().getPageStore(ADDRESS); pgStoreAddress.startPaging(); @@ -6035,7 +4761,7 @@ public class PagingTest extends ActiveMQTestBase { ClientProducer producer = session.createProducer(PagingTest.ADDRESS); - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 20; i++) { logger.debug("send message #{}", i); ClientMessage message = session.createMessage(true); @@ -6044,10 +4770,6 @@ public class PagingTest extends ActiveMQTestBase { message.setBodyInputStream(createFakeLargeStream(messageSize)); producer.send(message); - - if ((i + 1) % 2 == 0) { - session.commit(); - } } session.commit(); @@ -6057,7 +4779,7 @@ public class PagingTest extends ActiveMQTestBase { ClientConsumer cons = session.createConsumer(ADDRESS); for (int msgNr = 0; msgNr < 2; msgNr++) { - for (int i = 0; i < 5; i++) { + for (int i = 0; i < 2; i++) { ClientMessage msg = cons.receive(5000); assertNotNull(msg); @@ -6074,9 +4796,9 @@ public class PagingTest extends ActiveMQTestBase { pgStoreDLA.startPaging(); } - for (int i = 2; i < 100; i++) { + for (int i = 2; i < 20; i++) { logger.debug("Received message {}", i); - ClientMessage message = cons.receive(5000); + ClientMessage message = cons.receive(1000); assertNotNull("Message " + i + " wasn't received", message); message.acknowledge(); @@ -6136,10 +4858,10 @@ public class PagingTest extends ActiveMQTestBase { cons = session.createConsumer(ADDRESS); - for (int i = 2; i < 100; i++) { + for (int i = 2; i < 20; i++) { logger.debug("Received message {}", i); ClientMessage message = cons.receive(5000); - assertNotNull(message); + assertNotNull("message " + i + " should not be null", message); assertEquals("str" + i, message.getStringProperty("id")); @@ -6183,20 +4905,13 @@ public class PagingTest extends ActiveMQTestBase { assertNull(cons.receiveImmediate()); - long timeout = System.currentTimeMillis() + 5000; - pgStoreAddress = server.getPagingManager().getPageStore(ADDRESS); pgStoreAddress.getCursorProvider().getSubscription(serverQueue.getID()).cleanupEntries(false); PageCursorProviderAccessor.cleanup(pgStoreAddress.getCursorProvider()); - while (timeout > System.currentTimeMillis() && pgStoreAddress.isPaging()) { - Thread.sleep(50); - } - - assertFalse(pgStoreAddress.isPaging()); - + Wait.assertFalse(pgStoreAddress::isPaging); session.commit(); } finally { session.close(); @@ -6211,12 +4926,13 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testExpireLargeMessageOnPaging() throws Exception { - clearDataRecreateServerDirs(); + + int numberOfMessages = 100; Configuration config = createDefaultInVMConfig().setMessageExpiryScanPeriod(500).setJournalSyncNonTransactional(false); Map settings = new HashMap<>(); - AddressSettings dla = new AddressSettings().setMaxDeliveryAttempts(5).setDeadLetterAddress(new SimpleString("DLA")).setExpiryAddress(new SimpleString("DLA")); + AddressSettings dla = new AddressSettings().setMaxDeliveryAttempts(5).setDeadLetterAddress(new SimpleString("DLA")).setExpiryAddress(new SimpleString("DLA")).setMaxReadPageBytes(-1); settings.put(ADDRESS.toString(), dla); server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX, -1, -1, settings); @@ -6238,20 +4954,19 @@ public class PagingTest extends ActiveMQTestBase { PagingStore pgStoreAddress = server.getPagingManager().getPageStore(ADDRESS); pgStoreAddress.startPaging(); - PagingStore pgStoreDLA = server.getPagingManager().getPageStore(new SimpleString("DLA")); ClientProducer producer = session.createProducer(PagingTest.ADDRESS); ClientMessage message = null; - for (int i = 0; i < 500; i++) { - if (i % 100 == 0) + for (int i = 0; i < numberOfMessages; i++) { + if (i % 10 == 0) logger.debug("send message #{}", i); message = session.createMessage(true); message.putStringProperty("id", "str" + i); - message.setExpiration(System.currentTimeMillis() + 2000); + message.setExpiration(System.currentTimeMillis() + 200); if (i % 2 == 0) { message.setBodyInputStream(createFakeLargeStream(messageSize)); @@ -6265,11 +4980,8 @@ public class PagingTest extends ActiveMQTestBase { producer.send(message); - if ((i + 1) % 2 == 0) { + if ((i + 1) % 10 == 0) { session.commit(); - if (i < 400) { - pgStoreAddress.forceAnotherPage(); - } } } @@ -6281,7 +4993,7 @@ public class PagingTest extends ActiveMQTestBase { server.stop(); - Thread.sleep(3000); + Thread.sleep(1000); server.start(); @@ -6295,13 +5007,13 @@ public class PagingTest extends ActiveMQTestBase { ClientConsumer consAddr = session.createConsumer(ADDRESS); - assertNull(consAddr.receive(1000)); + assertNull(consAddr.receiveImmediate()); ClientConsumer cons = session.createConsumer("DLA"); - for (int i = 0; i < 500; i++) { - logger.debug("Received message {}", i); - message = cons.receive(10000); + for (int i = 0; i < numberOfMessages; i++) { + logger.debug("Received message {}", i); + message = cons.receive(1000); assertNotNull(message); message.acknowledge(); @@ -6335,21 +5047,20 @@ public class PagingTest extends ActiveMQTestBase { /** * When running this test from an IDE add this to the test command line so that the AssertionLoggerHandler works properly: - * - * -Dlog4j2.configurationFile=file:/tests/config/log4j2-tests-config.properties - * - * Note: Idea should get these from the pom and you shouldn't need to do this. + *

+ * -Dlog4j2.configurationFile=file:/tests/config/log4j2-tests-config.properties + *

+ * Note: Idea should get these from the pom and you shouldn't need to do this. */ @Test public void testFailMessagesNonDurable() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig(); HashMap settings = new HashMap<>(); - AddressSettings set = new AddressSettings(); - set.setAddressFullMessagePolicy(AddressFullMessagePolicy.FAIL); + AddressSettings set = new AddressSettings().setMaxReadPageBytes(-1); + set.setAddressFullMessagePolicy(AddressFullMessagePolicy.FAIL).setMaxSizeMessages(10); settings.put(PagingTest.ADDRESS.toString(), set); @@ -6419,15 +5130,15 @@ public class PagingTest extends ActiveMQTestBase { HashMap settings = new HashMap<>(); - AddressSettings set = new AddressSettings(); - set.setAddressFullMessagePolicy(AddressFullMessagePolicy.FAIL); - - settings.put(PagingTest.ADDRESS.toString(), set); - server = createServer(true, config, 1024, 5 * 1024, settings); server.start(); + server.getAddressSettingsRepository().clear(); + AddressSettings set = new AddressSettings().setMaxReadPageBytes(-1); + set.setAddressFullMessagePolicy(AddressFullMessagePolicy.FAIL).setMaxSizeMessages(11); + server.getAddressSettingsRepository().addMatch(PagingTest.ADDRESS.toString(), set); + locator.setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); sf = createSessionFactory(locator); @@ -6452,7 +5163,7 @@ public class PagingTest extends ActiveMQTestBase { // The address will actually fill up after 3 messages. Also, it takes 32 messages for the client's // credits to run out. for (int i = 0; i < 50; i++) { - if (i > 2) { + if (i > 10) { validateExceptionOnSending(producer, message); } else { producer.send(message); @@ -6485,14 +5196,13 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testFailMessagesDuplicates() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig(); HashMap settings = new HashMap<>(); - AddressSettings set = new AddressSettings(); - set.setAddressFullMessagePolicy(AddressFullMessagePolicy.FAIL); + AddressSettings set = new AddressSettings().setMaxReadPageBytes(-1); + set.setAddressFullMessagePolicy(AddressFullMessagePolicy.FAIL).setMaxSizeMessages(3); settings.put(PagingTest.ADDRESS.toString(), set); @@ -6757,10 +5467,6 @@ public class PagingTest extends ActiveMQTestBase { ClientConsumer consumerQ2 = session.createConsumer("Q2"); session.start(); - // consuming now - - // initial burst - for (int i = 0; i < initialBurst; i++) { ClientMessage m = consumerQ1.receive(5000); assertNotNull(m); @@ -6827,7 +5533,6 @@ public class PagingTest extends ActiveMQTestBase { // and expect it to move to the next page @Test public void testPageHole() throws Throwable { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -6919,98 +5624,7 @@ public class PagingTest extends ActiveMQTestBase { internalTestMultiFilters(false); } - @Test - public void testPageEmptyFile() throws Exception { - boolean persistentMessages = true; - - clearDataRecreateServerDirs(); - - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); - - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX, -1, -1); - - server.start(); - - final int messageSize = 1024; - - final int numberOfMessages = 100; - - try { - ServerLocator locator = createInVMNonHALocator().setClientFailureCheckPeriod(120000).setConnectionTTL(5000000).setCallTimeout(120000).setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); - - ClientSessionFactory sf = locator.createSessionFactory(); - - ClientSession session = sf.createSession(false, false, false); - - session.createQueue(new QueueConfiguration(PagingTest.ADDRESS)); - - PagingStore store = server.getPagingManager().getPageStore(ADDRESS); - store.forceAnotherPage(); - store.forceAnotherPage(); - - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); - - ClientMessage message = null; - - byte[] body = new byte[messageSize]; - - for (int i = 0; i < numberOfMessages; i++) { - message = session.createMessage(persistentMessages); - - ActiveMQBuffer bodyLocal = message.getBodyBuffer(); - - bodyLocal.writeBytes(body); - - producer.send(message); - } - - session.commit(); - - Queue queue = server.locateQueue(PagingTest.ADDRESS); - Wait.assertEquals(numberOfMessages, queue::getMessageCount); - - store.forceAnotherPage(); - - session.start(); - - ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS); - - for (int i = 0; i < numberOfMessages; i++) { - message = consumer.receive(5000); - assertNotNull(message); - message.acknowledge(); - } - - session.commit(); - - assertNull(consumer.receiveImmediate()); - - consumer.close(); - - PageCursorProviderAccessor.cleanup(store.getCursorProvider()); - - Wait.assertEquals(0, queue::getMessageCount); - - PageCursorProviderAccessor.cleanup(store.getCursorProvider()); - - Wait.assertFalse(store::isPaging, 5000, 100); - - sf.close(); - - locator.close(); - - Wait.assertEquals(1L, store::getNumberOfPages, 5000, 100); - - } finally { - try { - server.stop(); - } catch (Throwable ignored) { - } - } - } - public void internalTestMultiFilters(boolean browsing) throws Throwable { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -7091,7 +5705,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testPendingACKOutOfOrder() throws Throwable { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -7117,11 +5730,12 @@ public class PagingTest extends ActiveMQTestBase { ClientMessage msg = session.createMessage(true); msg.putIntProperty("count", i); prod.send(msg); - session.commit(); if ((i + 1) % 5 == 0 && i < 50) { + session.commit(); store.forceAnotherPage(); } } + session.commit(); session.start(); @@ -7173,7 +5787,6 @@ public class PagingTest extends ActiveMQTestBase { // Test a scenario where a page was complete and now needs to be cleared @Test public void testPageCompleteWasLive() throws Throwable { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -7282,7 +5895,6 @@ public class PagingTest extends ActiveMQTestBase { // Test a scenario where a page was complete and now needs to be cleared @Test public void testMoveMessages() throws Throwable { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -7290,13 +5902,13 @@ public class PagingTest extends ActiveMQTestBase { server.start(); - final int LARGE_MESSAGE_SIZE = 1024 * 1024; + final int LARGE_MESSAGE_SIZE = database == Database.JOURNAL ? 1024 * 1024 : 1024; try { ServerLocator locator = createInVMNonHALocator(); locator.setBlockOnDurableSend(false); ClientSessionFactory sf = locator.createSessionFactory(); - ClientSession session = sf.createSession(true, true, 0); + ClientSession session = sf.createSession(false, false, 0); session.createQueue(new QueueConfiguration("Q1")); session.createQueue(new QueueConfiguration("Q2")); @@ -7364,6 +5976,7 @@ public class PagingTest extends ActiveMQTestBase { msg.acknowledge(); assertEquals(i, msg.getIntProperty("count").intValue()); } + session.commit(); assertNull(cons.receiveImmediate()); @@ -7381,7 +5994,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testOnlyOnePageOnServerCrash() throws Throwable { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig(); @@ -7411,7 +6023,7 @@ public class PagingTest extends ActiveMQTestBase { } } - if (storeType == StoreConfiguration.StoreType.DATABASE) { + if (database != Database.JOURNAL) { server = new ActiveMQServerImpl(config, ManagementFactory.getPlatformMBeanServer(), new ActiveMQSecurityManagerImpl()) { @Override protected PagingStoreFactoryDatabase getPagingStoreFactory() throws Exception { @@ -7439,7 +6051,7 @@ public class PagingTest extends ActiveMQTestBase { addServer(server); - AddressSettings defaultSetting = new AddressSettings().setPageSizeBytes(PagingTest.PAGE_SIZE).setMaxSizeBytes(PagingTest.PAGE_SIZE + MESSAGE_SIZE).setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE); + AddressSettings defaultSetting = new AddressSettings().setPageSizeBytes(PagingTest.PAGE_SIZE).setMaxSizeBytes(PagingTest.PAGE_SIZE + MESSAGE_SIZE).setAddressFullMessagePolicy(AddressFullMessagePolicy.PAGE).setMaxReadPageBytes(-1); server.getAddressSettingsRepository().addMatch("#", defaultSetting); @@ -7519,7 +6131,6 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testPagingStoreDestroyed() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); @@ -7599,8 +6210,6 @@ public class PagingTest extends ActiveMQTestBase { private void testStopPagingWithoutConsumersOnOneQueue(boolean forceAnotherPage) throws Exception { boolean persistentMessages = true; - clearDataRecreateServerDirs(); - Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); @@ -7670,11 +6279,10 @@ public class PagingTest extends ActiveMQTestBase { @Test public void testStopPagingWithoutMsgsOnOneQueue() throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX, -1, -1); + server = createServer(true, config, PagingTest.PAGE_SIZE, 1, -1, -1); server.start(); @@ -7717,20 +6325,14 @@ public class PagingTest extends ActiveMQTestBase { ActiveMQBuffer bodyLocal = message.getBodyBuffer(); bodyLocal.writeBytes(body); producer.send(message); - if (i % 1000 == 0) { - session.commit(); - } + session.commit(); } - session.commit(); assertTrue(Arrays.asList(server.getPagingManager().getStoreNames()).contains(PagingTest.ADDRESS)); assertTrue(server.getPagingManager().getPageStore(PagingTest.ADDRESS).isPaging()); for (int i = 0; i < numberOfMessages; i++) { - ClientMessage msg = consumer1.receive(1000); + ClientMessage msg = consumer1.receive(5000); assertNotNull(msg); msg.acknowledge(); - if (i % 500 == 0) { - session.commit(); - } } session.commit(); assertNull(consumer1.receiveImmediate()); @@ -7752,7 +6354,6 @@ public class PagingTest extends ActiveMQTestBase { // We send messages to page, evict live page cache, send last message when mid consumed, and expect to receive all messages @Test public void testLivePageCacheEvicted() throws Throwable { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX); @@ -7781,7 +6382,7 @@ public class PagingTest extends ActiveMQTestBase { ClientMessage msgReceivedCons = null; // simulate the live page cache evicted for (int i = 0; i < num; i++) { - msgReceivedCons = cons.receive(1000); + msgReceivedCons = cons.receive(5000); assertNotNull(msgReceivedCons); assertTrue(msgReceivedCons.getIntProperty("index") == i); msgReceivedCons.acknowledge(); @@ -7795,7 +6396,7 @@ public class PagingTest extends ActiveMQTestBase { } } - msgReceivedCons = cons.receive(1000); + msgReceivedCons = cons.receive(5000); assertNotNull(msgReceivedCons); assertTrue(msgReceivedCons.getIntProperty("index") == num); msgReceivedCons.acknowledge(); @@ -7822,7 +6423,6 @@ public class PagingTest extends ActiveMQTestBase { } private void testRollbackPageTransaction(boolean rollbackBeforeDelivery) throws Exception { - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig(); @@ -7871,7 +6471,6 @@ public class PagingTest extends ActiveMQTestBase { session.close(); } - @Override protected void applySettings(ActiveMQServer server, final Configuration configuration, @@ -7883,13 +6482,72 @@ public class PagingTest extends ActiveMQTestBase { server.getConfiguration().setAddressQueueScanPeriod(100); } - @Override - protected Configuration createDefaultConfig(final int serverID, final boolean netty) throws Exception { - Configuration configuration = super.createDefaultConfig(serverID, netty); - if (storeType == StoreConfiguration.StoreType.DATABASE) { - setDBStoreType(configuration); + @Test + public void testSimpleNoTXSend() throws Exception { + Configuration config = createDefaultNettyConfig(); + + server = createServer(true, config, PAGE_SIZE, PAGE_MAX, -1, -1); + server.start(); + + internalNoTX("CORE"); + internalNoTX("OPENWIRE"); + internalNoTX("AMQP"); + } + + private void internalNoTX(String protocol) throws Exception { + int numberOfMessages = 20; + + String queueName = "TEST" + RandomUtil.randomString(); + + try { + server.addAddressInfo(new AddressInfo(queueName).addRoutingType(RoutingType.ANYCAST)); + server.createQueue(new QueueConfiguration(queueName).setRoutingType(RoutingType.ANYCAST)); + } catch (Exception ignored) { + } + + Wait.waitFor(() -> server.locateQueue(queueName) != null); + Queue testQueue = server.locateQueue(queueName); + + testQueue.getPagingStore().startPaging(); + Assert.assertTrue(testQueue.getPagingStore().isPaging()); + + ConnectionFactory connectionFactory = CFUtil.createConnectionFactory(protocol, "tcp://localhost:61616"); + try (Connection connection = connectionFactory.createConnection()) { + Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE); + MessageProducer producer = session.createProducer(session.createQueue(queueName)); + for (int i = 0; i < numberOfMessages; i++) { + logger.debug("Sent {}", i); + producer.send(session.createTextMessage("Hello" + i)); + } + } + + server.stop(); + server = createServer(createDefaultConfig(0, true)); + server.start(); + + Queue queue = server.locateQueue(queueName); + Wait.assertEquals(numberOfMessages, queue::getMessageCount); + + receiveMessages(connectionFactory, queueName, numberOfMessages); + } + + private void receiveMessages(ConnectionFactory connectionFactory, + String queueName, + int numberOfMessages) throws JMSException { + try (Connection connection = connectionFactory.createConnection()) { + connection.start(); + Session session = connection.createSession(false, Session.AUTO_ACKNOWLEDGE); + MessageConsumer consumer = session.createConsumer(session.createQueue(queueName)); + for (int i = 0; i < numberOfMessages; i++) { + if (i % 100 == 0) { + logger.debug("Received {}", i); + } + TextMessage message = (TextMessage) consumer.receive(5000); + Assert.assertNotNull(message); + Assert.assertEquals("Hello" + i, message.getText()); + } + Assert.assertNull(consumer.receiveNoWait()); } - return configuration; } private static final class DummyOperationContext implements OperationContext { @@ -7954,4 +6612,5 @@ public class PagingTest extends ActiveMQTestBase { runnable.done(); } } + } diff --git a/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/PrintDataTest.java b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/PrintDataTest.java new file mode 100644 index 0000000000..2adcca2b26 --- /dev/null +++ b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/PrintDataTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.tests.db.paging; + +import javax.jms.Connection; +import javax.jms.ConnectionFactory; +import javax.jms.MessageProducer; +import javax.jms.Session; +import javax.jms.TextMessage; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Collection; + +import org.apache.activemq.artemis.api.core.QueueConfiguration; +import org.apache.activemq.artemis.api.core.RoutingType; +import org.apache.activemq.artemis.cli.commands.tools.PrintData; +import org.apache.activemq.artemis.core.server.ActiveMQServer; +import org.apache.activemq.artemis.core.server.Queue; +import org.apache.activemq.artemis.core.server.impl.AddressInfo; +import org.apache.activemq.artemis.tests.db.common.Database; +import org.apache.activemq.artemis.tests.db.common.ParameterDBTestBase; +import org.apache.activemq.artemis.tests.util.CFUtil; +import org.apache.activemq.artemis.utils.RandomUtil; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runners.Parameterized; + +public class PrintDataTest extends ParameterDBTestBase { + + ActiveMQServer server; + + @Parameterized.Parameters(name = "db={0}") + public static Collection parameters() { + return convertParameters(Database.selectedList()); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + server = createServer(createDefaultConfig(0, true)); + server.start(); + + } + + @Test + public void testData() throws Exception { + + String queueName = RandomUtil.randomString(); + server.addAddressInfo(new AddressInfo(queueName).addRoutingType(RoutingType.ANYCAST)); + Queue queue = server.createQueue(new QueueConfiguration().setAddress(queueName).setName(queueName).setDurable(true).setRoutingType(RoutingType.ANYCAST)); + queue.getPagingStore().startPaging(); + + int numberOfMessages = 10; + + ConnectionFactory cf = CFUtil.createConnectionFactory("core", "tcp://localhost:61616"); + try (Connection connection = cf.createConnection()) { + Session session = connection.createSession(true, Session.SESSION_TRANSACTED); + MessageProducer producer = session.createProducer(session.createQueue(queueName)); + + for (int i = 0; i < numberOfMessages; i++) { + TextMessage message = session.createTextMessage("message " + i); + message.setStringProperty("i", "message " + i); + producer.send(message); + } + session.commit(); + } + server.stop(); + + PrintData printData = new PrintData(); + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + PrintStream printStream = new PrintStream(byteArrayOutputStream, true, StandardCharsets.UTF_8.name()); + + printData.printDataJDBC(server.getConfiguration(), printStream); + + String printDataOutput = byteArrayOutputStream.toString(); + + for (int i = 0; i < numberOfMessages; i++) { + Assert.assertTrue(printDataOutput.lastIndexOf("message " + i) >= 0); + } + // I know this is a bit fragile, but the queues routed portion of the report was not working. + // if the report ever changes, so the test will need to be changed. + Assert.assertTrue(printDataOutput.lastIndexOf("queues routed") >= 0); + + } + +} diff --git a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/DatabasePagingTest.java b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/RealServerDatabasePagingTest.java similarity index 74% rename from tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/DatabasePagingTest.java rename to tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/RealServerDatabasePagingTest.java index 9d3678f0aa..65a42ad01a 100644 --- a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/DatabasePagingTest.java +++ b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/RealServerDatabasePagingTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.activemq.artemis.tests.soak.paging; +package org.apache.activemq.artemis.tests.db.paging; import javax.jms.BytesMessage; import javax.jms.Connection; @@ -25,70 +25,45 @@ import javax.jms.MessageProducer; import javax.jms.Queue; import javax.jms.Session; import java.lang.invoke.MethodHandles; -import java.util.ArrayList; import java.util.Collection; import java.util.concurrent.TimeUnit; -import org.apache.activemq.artemis.tests.soak.SoakTestBase; +import org.apache.activemq.artemis.tests.db.common.Database; +import org.apache.activemq.artemis.tests.db.common.ParameterDBTestBase; import org.apache.activemq.artemis.tests.util.CFUtil; -import org.apache.activemq.artemis.tests.util.RandomUtil; +import org.apache.activemq.artemis.utils.RandomUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.activemq.artemis.tests.soak.TestParameters.testProperty; +import static org.apache.activemq.artemis.utils.TestParameters.testProperty; -@RunWith(Parameterized.class) -public class DatabasePagingTest extends SoakTestBase { +public class RealServerDatabasePagingTest extends ParameterDBTestBase { private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String TEST_NAME = "PGDB"; - // you can use ./start-${database}-podman.sh scripts from ./src/test/scripts to start the databases. - // support values are derby, mysql and postgres - private static final String DB_LIST = testProperty(TEST_NAME, "DB_LIST", "derby"); - private static final int MAX_MESSAGES = Integer.parseInt(testProperty(TEST_NAME, "MAX_MESSAGES", "200")); - private static final int MESSAGE_SIZE = Integer.parseInt(testProperty(TEST_NAME, "MESSAGE_SIZE", "200")); + private static final int MESSAGE_SIZE = Integer.parseInt(testProperty(TEST_NAME, "MESSAGE_SIZE", "1000")); private static final int COMMIT_INTERVAL = Integer.parseInt(testProperty(TEST_NAME, "COMMIT_INTERVAL", "100")); Process serverProcess; - final String database; - - final String serverName; - - @Parameterized.Parameters(name = "protocol={0}") + @Parameterized.Parameters(name = "db={0}") public static Collection parameters() { - String[] protocols = DB_LIST.split(","); - - ArrayList parameters = new ArrayList<>(); - for (String str : protocols) { - logger.info("Adding {} to the list for the test", str); - parameters.add(new Object[]{str}); - } - - return parameters; - } - - public DatabasePagingTest(String database) { - this.database = database; - serverName = "database-paging/" + database; + return convertParameters(Database.selectedList()); } @Before public void before() throws Exception { - cleanupData(serverName); - - serverProcess = startServer(serverName, 0, 60_000); + serverProcess = startServer(database.getName(), 0, 60_000); } @@ -106,7 +81,6 @@ public class DatabasePagingTest extends SoakTestBase { ConnectionFactory connectionFactory = CFUtil.createConnectionFactory(protocol, "tcp://localhost:61616"); - try (Connection connection = connectionFactory.createConnection()) { byte[] messageLoad = new byte[MESSAGE_SIZE]; Session session = connection.createSession(true, Session.SESSION_TRANSACTED); @@ -129,8 +103,7 @@ public class DatabasePagingTest extends SoakTestBase { serverProcess.waitFor(1, TimeUnit.MINUTES); Assert.assertFalse(serverProcess.isAlive()); - serverProcess = startServer(serverName, 0, 60_000); - + serverProcess = startServer(database.getName(), 0, 60_000); try (Connection connection = connectionFactory.createConnection()) { connection.start(); diff --git a/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/SchemaValidationTest.java b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/SchemaValidationTest.java new file mode 100644 index 0000000000..27e9bf3ccc --- /dev/null +++ b/tests/db-tests/src/test/java/org/apache/activemq/artemis/tests/db/paging/SchemaValidationTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.activemq.artemis.tests.db.paging; + +import java.util.Collection; + +import org.apache.activemq.artemis.core.config.Configuration; +import org.apache.activemq.artemis.core.config.storage.DatabaseStorageConfiguration; +import org.apache.activemq.artemis.core.server.ActiveMQServer; +import org.apache.activemq.artemis.logs.AssertionLoggerHandler; +import org.apache.activemq.artemis.tests.db.common.Database; +import org.apache.activemq.artemis.tests.db.common.ParameterDBTestBase; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runners.Parameterized; + +public class SchemaValidationTest extends ParameterDBTestBase { + + @Parameterized.Parameters(name = "db={0}") + public static Collection parameters() { + return convertParameters(Database.selectedList()); + } + + @Test + public void testTableNameTooLong() throws Exception { + + try (AssertionLoggerHandler loggerHandler = new AssertionLoggerHandler(true)) { + final Configuration config = createDefaultInVMConfig(); + final DatabaseStorageConfiguration storageConfiguration = (DatabaseStorageConfiguration) config.getStoreConfiguration(); + //set the page store table to be longer than 10 chars -> the paging manager initialization will fail + storageConfiguration.setPageStoreTableName("PAGE_STORE_"); + + final int PAGE_MAX = 20 * 1024; + + final int PAGE_SIZE = 10 * 1024; + + final ActiveMQServer server = createServer(true, config, PAGE_SIZE, PAGE_MAX); + server.start(); + + //due to a failed initialisation of the paging manager, it must be null + Assert.assertNull(server.getPagingManager()); + + server.stop(); + + Assert.assertTrue(loggerHandler.findText("AMQ224000")); // Failure in initialization + } + } + +} diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/GlobalPagingTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/GlobalJournalPagingTest.java similarity index 92% rename from tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/GlobalPagingTest.java rename to tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/GlobalJournalPagingTest.java index 23439ede27..280e5d2cc7 100644 --- a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/GlobalPagingTest.java +++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/GlobalJournalPagingTest.java @@ -40,7 +40,6 @@ import org.apache.activemq.artemis.api.core.client.ServerLocator; import org.apache.activemq.artemis.api.core.management.ManagementHelper; import org.apache.activemq.artemis.core.client.impl.ClientMessageImpl; import org.apache.activemq.artemis.core.config.Configuration; -import org.apache.activemq.artemis.core.config.StoreConfiguration; import org.apache.activemq.artemis.core.paging.PagingManager; import org.apache.activemq.artemis.core.server.ActiveMQServer; import org.apache.activemq.artemis.core.server.MessageReference; @@ -50,19 +49,10 @@ import org.apache.activemq.artemis.core.settings.impl.AddressFullMessagePolicy; import org.apache.activemq.artemis.core.settings.impl.AddressSettings; import org.apache.activemq.artemis.tests.util.Wait; import org.junit.Assert; -import org.junit.Assume; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -@RunWith(Parameterized.class) -public class GlobalPagingTest extends PagingTest { - - public GlobalPagingTest(StoreConfiguration.StoreType storeType) { - super(storeType); - } +public class GlobalJournalPagingTest extends JournalPagingTest { @Override @Before @@ -95,20 +85,13 @@ public class GlobalPagingTest extends PagingTest { server.getAddressSettingsRepository().addMatch("#", defaultSetting); } - // test doesn't make sense on GlobalPaging due to configuration issues - @Test @Ignore @Override - public void testPurge() throws Exception { - } - @Test public void testPagingOverFullDisk() throws Exception { - Assume.assumeTrue(storeType != StoreConfiguration.StoreType.DATABASE); - clearDataRecreateServerDirs(); Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); - server = createServer(true, config, PagingTest.PAGE_SIZE, PagingTest.PAGE_MAX, -1, -1, new HashMap<>()); + server = createServer(true, config, JournalPagingTest.PAGE_SIZE, JournalPagingTest.PAGE_MAX, -1, -1, new HashMap<>()); Assert.assertTrue(customServerCreated); server.getConfiguration().setGlobalMaxSize(-1); server.getConfiguration().setAddressQueueScanPeriod(100); @@ -127,9 +110,9 @@ public class GlobalPagingTest extends PagingTest { final ClientSession session = sf.createSession(false, false, false); - session.createQueue(new QueueConfiguration(PagingTest.ADDRESS)); + session.createQueue(new QueueConfiguration(JournalPagingTest.ADDRESS)); - final ClientProducer producer = session.createProducer(PagingTest.ADDRESS); + final ClientProducer producer = session.createProducer(JournalPagingTest.ADDRESS); ClientMessage message = null; @@ -182,7 +165,7 @@ public class GlobalPagingTest extends PagingTest { // The consumer has to be created after the getMessageCount(queue) assertion // otherwise delivery could alter the messagecount and give us a false failure - ClientConsumer consumer = session.createConsumer(PagingTest.ADDRESS); + ClientConsumer consumer = session.createConsumer(JournalPagingTest.ADDRESS); ClientMessage msg = null; for (int i = 0; i < numberOfMessages * 2; i++) { @@ -224,7 +207,7 @@ public class GlobalPagingTest extends PagingTest { Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); - final ActiveMQServer server = createServer(true, config, PagingTest.PAGE_SIZE, -1); + final ActiveMQServer server = createServer(true, config, JournalPagingTest.PAGE_SIZE, -1); try { final SimpleString managementAddress = server.getConfiguration().getManagementAddress(); @@ -311,7 +294,7 @@ public class GlobalPagingTest extends PagingTest { Configuration config = createDefaultNettyConfig().setJournalSyncNonTransactional(false); - final ActiveMQServer server = createServer(true, config, PagingTest.PAGE_SIZE, -1); + final ActiveMQServer server = createServer(true, config, JournalPagingTest.PAGE_SIZE, -1); try { final SimpleString managementAddress = server.getConfiguration().getManagementAddress(); diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/JournalPagingTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/JournalPagingTest.java new file mode 100644 index 0000000000..6f282f2385 --- /dev/null +++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/JournalPagingTest.java @@ -0,0 +1,762 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.activemq.artemis.tests.integration.paging; + +import javax.jms.Connection; +import javax.jms.MessageConsumer; +import javax.jms.MessageProducer; +import javax.jms.Session; +import javax.transaction.xa.XAResource; +import javax.transaction.xa.Xid; +import java.io.File; +import java.io.FileOutputStream; +import java.io.PrintWriter; +import java.lang.invoke.MethodHandles; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.activemq.artemis.api.core.ActiveMQBuffer; +import org.apache.activemq.artemis.api.core.QueueConfiguration; +import org.apache.activemq.artemis.api.core.RoutingType; +import org.apache.activemq.artemis.api.core.SimpleString; +import org.apache.activemq.artemis.api.core.client.ClientConsumer; +import org.apache.activemq.artemis.api.core.client.ClientMessage; +import org.apache.activemq.artemis.api.core.client.ClientProducer; +import org.apache.activemq.artemis.api.core.client.ClientSession; +import org.apache.activemq.artemis.api.core.client.ClientSessionFactory; +import org.apache.activemq.artemis.api.core.client.ServerLocator; +import org.apache.activemq.artemis.core.config.Configuration; +import org.apache.activemq.artemis.core.paging.PagedMessage; +import org.apache.activemq.artemis.core.paging.impl.Page; +import org.apache.activemq.artemis.core.paging.impl.PageTransactionInfoImpl; +import org.apache.activemq.artemis.core.paging.impl.PagingStoreFactoryNIO; +import org.apache.activemq.artemis.core.paging.impl.PagingStoreImpl; +import org.apache.activemq.artemis.core.persistence.StorageManager; +import org.apache.activemq.artemis.core.server.ActiveMQServer; +import org.apache.activemq.artemis.core.server.Queue; +import org.apache.activemq.artemis.core.server.impl.AddressInfo; +import org.apache.activemq.artemis.core.server.impl.QueueImpl; +import org.apache.activemq.artemis.core.settings.impl.AddressSettings; +import org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory; +import org.apache.activemq.artemis.logs.AssertionLoggerHandler; +import org.apache.activemq.artemis.tests.util.ActiveMQTestBase; +import org.apache.activemq.artemis.tests.util.RandomUtil; +import org.apache.activemq.artemis.tests.util.Wait; +import org.apache.activemq.artemis.utils.RetryRule; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JournalPagingTest extends ActiveMQTestBase { + + protected static final int PAGE_MAX = 100 * 1024; + protected static final int PAGE_SIZE = 10 * 1024; + static final int MESSAGE_SIZE = 1024; // 1k + static final SimpleString ADDRESS = new SimpleString("SimpleAddress"); + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + @Rule + public RetryRule retryMethod = new RetryRule(1); + protected ServerLocator locator; + protected ActiveMQServer server; + protected ClientSessionFactory sf; + private AssertionLoggerHandler loggerHandler; + + @Before + public void checkLoggerStart() throws Exception { + loggerHandler = new AssertionLoggerHandler(); + } + + @After + public void checkLoggerEnd() throws Exception { + try { + // These are the message errors for the negative size address size + Assert.assertFalse(loggerHandler.findText("222214")); + Assert.assertFalse(loggerHandler.findText("222215")); + } finally { + loggerHandler.close(); + } + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + locator = createInVMNonHALocator(); + } + + @Test + public void testPageCleanupWithInvalidDataTruncated() throws Exception { + testPageCleanupWithInvalidData(true); + } + + @Test + public void testPageCleanupWithInvalidData() throws Exception { + testPageCleanupWithInvalidData(false); + } + + public void testPageCleanupWithInvalidData(boolean truncated) throws Exception { + clearDataRecreateServerDirs(); + + Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); + + server = createServer(true, config, JournalPagingTest.PAGE_SIZE, JournalPagingTest.PAGE_MAX); + + server.start(); + + final int numberOfMessages = 100; + + locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); + + sf = createSessionFactory(locator); + + ClientSession session = sf.createSession(false, true, true); + + session.createQueue(new QueueConfiguration(JournalPagingTest.ADDRESS)); + + ClientProducer producer = session.createProducer(JournalPagingTest.ADDRESS); + + ClientMessage message = null; + + byte[] body = new byte[10]; + + ByteBuffer bb = ByteBuffer.wrap(body); + + for (int j = 1; j <= 10; j++) { + bb.put(getSamplebyte(j)); + } + + Queue queue = server.locateQueue(ADDRESS); + queue.getPagingStore().startPaging(); + + queue.getPagingStore().forceAnotherPage(); // forcing an empty file, just to make it more challenging + + int page = 1; + for (int i = 0; i < numberOfMessages; i++) { + if (i % 10 == 0 && i > 0) { + queue.getPagingStore().forceAnotherPage(); + page++; + } + message = session.createMessage(true); + + ActiveMQBuffer bodyLocal = message.getBodyBuffer(); + + bodyLocal.writeBytes(body); + + message.putIntProperty("i", i); + message.putIntProperty("page", page); + + producer.send(message); + } + + queue.getPagingStore().getCursorProvider().disableCleanup(); + + ClientConsumer consumer = session.createConsumer(ADDRESS); + session.start(); + + for (int i = 0; i < 11; i++) { + ClientMessage msgRec = consumer.receive(1000); + Assert.assertNotNull(msgRec); + msgRec.acknowledge(); + } + session.commit(); + + consumer.close(); + + consumer = session.createConsumer(ADDRESS, SimpleString.toSimpleString("i=29")); + + message = consumer.receive(5000); + Assert.assertNotNull(message); + message.acknowledge(); + session.commit(); + + File folder = queue.getPagingStore().getFolder(); + + // We will truncate two files + for (int f = 2; f <= 3; f++) { + String fileName = ((PagingStoreImpl) queue.getPagingStore()).createFileName(f); + File file = new File(folder, fileName); + file.delete(); + file.createNewFile(); + if (!truncated) { + FileOutputStream fileOutputStream = new FileOutputStream(file); + fileOutputStream.write(new byte[10]); + fileOutputStream.close(); + } + } + sf.close(); + + server.getStorageManager().getMessageJournal().scheduleCompactAndBlock(5000); + + Page page4 = queue.getPagingStore().newPageObject(4); + page4.open(true); + org.apache.activemq.artemis.utils.collections.LinkedList messagesRead = page4.read(server.getStorageManager()); + Assert.assertEquals(10, messagesRead.size()); + page4.close(false); + page4.delete(null); + page4.open(true); + for (int i = 0; i < 9; i++) { + page4.write(messagesRead.get(i)); // this will make message 29 disappear + } + page4.close(false); + + server.stop(); + + server.start(); + + queue = server.locateQueue(ADDRESS); + Assert.assertTrue(queue.getPagingStore().isPaging()); + + queue.getPageSubscription().enableAutoCleanup(); // this should been true already as the server was restarted, just braces and belts + + sf = createSessionFactory(locator); + session = sf.createSession(false, true, true); + + logger.info("*******************************************************************************************************************************"); + logger.info("Creating consumer"); + + consumer = session.createConsumer(ADDRESS); + session.start(); + + for (int i = 20; i < numberOfMessages; i++) { // I made one message disappear on page 4 + if (i != 29) { // I made message 29 disappear + ClientMessage msgClient = consumer.receive(1000); + Assert.assertNotNull(msgClient); + Assert.assertEquals(i, msgClient.getIntProperty("i").intValue()); + msgClient.acknowledge(); + } + } + ClientMessage msgClient = consumer.receiveImmediate(); + Assert.assertNull(msgClient); + session.commit(); + + Wait.assertFalse(queue.getPagingStore()::isPaging, 5000, 100); + } + + @Test + public void testEmptyAddress() throws Exception { + clearDataRecreateServerDirs(); + + Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); + + server = createServer(true, config, JournalPagingTest.PAGE_SIZE, JournalPagingTest.PAGE_MAX); + + server.start(); + + final int numberOfMessages = 500; + + locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); + + sf = createSessionFactory(locator); + + ClientSession session = sf.createSession(false, false, false); + + session.createQueue(new QueueConfiguration(JournalPagingTest.ADDRESS).setRoutingType(RoutingType.ANYCAST)); + + ClientProducer producer = session.createProducer(JournalPagingTest.ADDRESS); + + byte[] body = new byte[MESSAGE_SIZE]; + + ByteBuffer bb = ByteBuffer.wrap(body); + + for (int j = 1; j <= MESSAGE_SIZE; j++) { + bb.put(getSamplebyte(j)); + } + + for (int i = 0; i < numberOfMessages; i++) { + ClientMessage message = session.createMessage(true); + + message.getBodyBuffer().writeBytes(body); + + producer.send(message); + if (i % 1000 == 0) { + session.commit(); + } + } + session.commit(); + producer.close(); + session.close(); + + String addressTxt = server.getPagingManager().getPageStore(JournalPagingTest.ADDRESS).getFolder().getAbsolutePath() + File.separator + PagingStoreFactoryNIO.ADDRESS_FILE; + + server.stop(); + + new PrintWriter(addressTxt).close(); + Assert.assertTrue(new File(addressTxt).exists()); + + final AtomicBoolean activationFailures = new AtomicBoolean(); + + server.registerActivationFailureListener(exception -> activationFailures.set(true)); + + server.start(); + + server.stop(); + + assertFalse(activationFailures.get()); + } + + @Test + public void testPurge() throws Exception { + clearDataRecreateServerDirs(); + + Configuration config = createDefaultNettyConfig().setJournalSyncNonTransactional(false); + + server = createServer(true, config, JournalPagingTest.PAGE_SIZE, JournalPagingTest.PAGE_MAX); + + server.start(); + + SimpleString queue = new SimpleString("testPurge:" + RandomUtil.randomString()); + server.addAddressInfo(new AddressInfo(queue, RoutingType.ANYCAST)); + QueueImpl purgeQueue = (QueueImpl) server.createQueue(new QueueConfiguration(queue).setRoutingType(RoutingType.ANYCAST).setMaxConsumers(1).setPurgeOnNoConsumers(true).setAutoCreateAddress(false)); + + ActiveMQConnectionFactory cf = new ActiveMQConnectionFactory(); + Connection connection = cf.createConnection(); + Session session = connection.createSession(true, Session.SESSION_TRANSACTED); + javax.jms.Queue jmsQueue = session.createQueue(queue.toString()); + + MessageProducer producer = session.createProducer(jmsQueue); + + for (int i = 0; i < 100; i++) { + producer.send(session.createTextMessage("hello" + i)); + } + session.commit(); + + Wait.assertEquals(0, purgeQueue::getMessageCount); + + Wait.assertEquals(0, purgeQueue.getPageSubscription().getPagingStore()::getAddressSize); + + MessageConsumer consumer = session.createConsumer(jmsQueue); + + for (int i = 0; i < 100; i++) { + producer.send(session.createTextMessage("hello" + i)); + if (i == 10) { + purgeQueue.getPageSubscription().getPagingStore().startPaging(); + } + } + session.commit(); + + consumer.close(); + + Wait.assertEquals(0, purgeQueue::getMessageCount); + + Wait.assertFalse(purgeQueue.getPageSubscription()::isPaging); + + Wait.assertEquals(0, purgeQueue.getPageSubscription().getPagingStore()::getAddressSize); + consumer = session.createConsumer(jmsQueue); + + for (int i = 0; i < 100; i++) { + purgeQueue.getPageSubscription().getPagingStore().startPaging(); + Assert.assertTrue(purgeQueue.getPageSubscription().isPaging()); + producer.send(session.createTextMessage("hello" + i)); + if (i % 2 == 0) { + session.commit(); + } + } + + session.commit(); + + Wait.assertTrue(purgeQueue.getPageSubscription()::isPaging); + + connection.start(); + + server.getStorageManager().getMessageJournal().scheduleCompactAndBlock(50000); + Assert.assertNotNull(consumer.receive(5000)); + session.commit(); + + consumer.close(); + + Wait.assertEquals(0, purgeQueue::getMessageCount); + Wait.assertEquals(0, purgeQueue.getPageSubscription().getPagingStore()::getAddressSize); + Wait.assertFalse(purgeQueue.getPageSubscription()::isPaging, 5000, 100); + + StorageManager sm = server.getStorageManager(); + + for (int i = 0; i < 1000; i++) { + long tx = sm.generateID(); + PageTransactionInfoImpl txinfo = new PageTransactionInfoImpl(tx); + sm.storePageTransaction(tx, txinfo); + sm.commit(tx); + tx = sm.generateID(); + sm.updatePageTransaction(tx, txinfo, 1); + sm.commit(tx); + } + + server.stop(); + server.start(); + Wait.assertEquals(0, () -> server.getPagingManager().getTransactions().size()); + } + + // First page is complete but it wasn't deleted + @Test + public void testPreparedACKRemoveAndRestart() throws Exception { + clearDataRecreateServerDirs(); + + Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false); + + server = createServer(true, config, JournalPagingTest.PAGE_SIZE, JournalPagingTest.PAGE_MAX); + + server.start(); + + final int numberOfMessages = 10; + + locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true).setAckBatchSize(0); + + sf = createSessionFactory(locator); + + ClientSession session = sf.createSession(false, true, true); + + session.createQueue(new QueueConfiguration(JournalPagingTest.ADDRESS)); + + Queue queue = server.locateQueue(JournalPagingTest.ADDRESS); + + ClientProducer producer = session.createProducer(JournalPagingTest.ADDRESS); + + byte[] body = new byte[MESSAGE_SIZE]; + + ByteBuffer bb = ByteBuffer.wrap(body); + + for (int j = 1; j <= MESSAGE_SIZE; j++) { + bb.put(getSamplebyte(j)); + } + + queue.getPageSubscription().getPagingStore().startPaging(); + + forcePage(queue); + + for (int i = 0; i < numberOfMessages; i++) { + ClientMessage message = session.createMessage(true); + + message.putIntProperty("count", i); + + ActiveMQBuffer bodyLocal = message.getBodyBuffer(); + + bodyLocal.writeBytes(body); + + producer.send(message); + + if (i == 4) { + session.commit(); + queue.getPageSubscription().getPagingStore().forceAnotherPage(); + } + } + + session.commit(); + + session.close(); + + session = sf.createSession(true, false, false); + + ClientConsumer cons = session.createConsumer(ADDRESS); + + session.start(); + + for (int i = 0; i <= 4; i++) { + Xid xidConsumeNoCommit = newXID(); + session.start(xidConsumeNoCommit, XAResource.TMNOFLAGS); + // First message is consumed, prepared, will be rolled back later + ClientMessage firstMessageConsumed = cons.receive(5000); + assertNotNull(firstMessageConsumed); + firstMessageConsumed.acknowledge(); + session.end(xidConsumeNoCommit, XAResource.TMSUCCESS); + session.prepare(xidConsumeNoCommit); + } + + File pagingFolder = queue.getPageSubscription().getPagingStore().getFolder(); + + server.stop(); + + // remove the very first page. a restart should not fail + File fileToRemove = new File(pagingFolder, "000000001.page"); + Assert.assertTrue(fileToRemove.delete()); + + server.start(); + + sf = createSessionFactory(locator); + + session = sf.createSession(false, true, true); + + cons = session.createConsumer(ADDRESS); + + session.start(); + + for (int i = 5; i < numberOfMessages; i++) { + ClientMessage message = cons.receive(1000); + assertNotNull(message); + assertEquals(i, message.getIntProperty("count").intValue()); + message.acknowledge(); + } + assertNull(cons.receiveImmediate()); + session.commit(); + } + + /** + * @param queue + * @throws InterruptedException + */ + private void forcePage(Queue queue) throws InterruptedException { + for (long timeout = System.currentTimeMillis() + 5000; timeout > System.currentTimeMillis() && !queue.getPageSubscription().getPagingStore().isPaging(); ) { + Thread.sleep(10); + } + assertTrue(queue.getPageSubscription().getPagingStore().isPaging()); + } + + @Test + public void testInabilityToCreateDirectoryDuringPaging() throws Exception { + + try (AssertionLoggerHandler loggerHandler = new AssertionLoggerHandler()) { + clearDataRecreateServerDirs(); + + Configuration config = createDefaultInVMConfig().setJournalSyncNonTransactional(false).setPagingDirectory("/" + UUID.randomUUID().toString()); + + server = createServer(true, config, JournalPagingTest.PAGE_SIZE, JournalPagingTest.PAGE_MAX); + + server.start(); + + final int numberOfMessages = 100; + + locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); + + sf = createSessionFactory(locator); + + ClientSession session = sf.createSession(false, true, true); + + session.createQueue(new QueueConfiguration(JournalPagingTest.ADDRESS)); + + ClientProducer producer = session.createProducer(JournalPagingTest.ADDRESS); + + ClientMessage message = null; + + byte[] body = new byte[MESSAGE_SIZE]; + + ByteBuffer bb = ByteBuffer.wrap(body); + + for (int j = 1; j <= MESSAGE_SIZE; j++) { + bb.put(getSamplebyte(j)); + } + + for (int i = 0; i < numberOfMessages; i++) { + message = session.createMessage(true); + + ActiveMQBuffer bodyLocal = message.getBodyBuffer(); + + bodyLocal.writeBytes(body); + + message.putIntProperty(new SimpleString("id"), i); + + try { + producer.send(message); + } catch (Exception e) { + // ignore + } + } + assertTrue(Wait.waitFor(() -> server.getState() == ActiveMQServer.SERVER_STATE.STOPPED, 5000, 200)); + session.close(); + sf.close(); + locator.close(); + } finally { + Assert.assertTrue(loggerHandler.findText("AMQ144010")); + } + } + + /** + * This test will remove all the page directories during a restart, simulating a crash scenario. The server should still start after this + */ + @Test + public void testDeletePhysicalPages() throws Exception { + clearDataRecreateServerDirs(); + + Configuration config = createDefaultInVMConfig().setPersistDeliveryCountBeforeDelivery(true); + + config.setJournalSyncNonTransactional(false); + + server = createServer(true, config, JournalPagingTest.PAGE_SIZE, JournalPagingTest.PAGE_MAX); + + server.start(); + + final int numberOfMessages = 300; + + locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); + + sf = createSessionFactory(locator); + + ClientSession session = sf.createSession(false, false, false); + + session.createQueue(new QueueConfiguration(JournalPagingTest.ADDRESS)); + + ClientProducer producer = session.createProducer(JournalPagingTest.ADDRESS); + + ClientMessage message = null; + + byte[] body = new byte[MESSAGE_SIZE]; + + ByteBuffer bb = ByteBuffer.wrap(body); + + for (int j = 1; j <= MESSAGE_SIZE; j++) { + bb.put(getSamplebyte(j)); + } + + for (int i = 0; i < numberOfMessages; i++) { + message = session.createMessage(true); + + ActiveMQBuffer bodyLocal = message.getBodyBuffer(); + + bodyLocal.writeBytes(body); + + message.putIntProperty(new SimpleString("id"), i); + + producer.send(message); + if (i % 1000 == 0) { + session.commit(); + } + } + session.commit(); + session.close(); + + session = null; + + sf.close(); + locator.close(); + + server.stop(); + + server = createServer(true, config, JournalPagingTest.PAGE_SIZE, JournalPagingTest.PAGE_MAX); + server.start(); + + locator = createInVMNonHALocator(); + sf = createSessionFactory(locator); + + Queue queue = server.locateQueue(ADDRESS); + + Wait.assertEquals(numberOfMessages, queue::getMessageCount); + + ClientSession sessionConsumer = sf.createSession(false, false, false); + sessionConsumer.start(); + ClientConsumer consumer = sessionConsumer.createConsumer(JournalPagingTest.ADDRESS); + for (int msgCount = 0; msgCount < numberOfMessages; msgCount++) { + logger.debug("Received {}", msgCount); + ClientMessage msg = consumer.receive(100); + if (msg == null) { + logger.debug("It's null. leaving now"); + sessionConsumer.commit(); + fail("Didn't receive a message"); + } + msg.acknowledge(); + + if (msgCount % 5 == 0) { + logger.debug("commit"); + sessionConsumer.commit(); + } + } + + sessionConsumer.commit(); + + sessionConsumer.close(); + + sf.close(); + + locator.close(); + + Wait.assertEquals(0, queue::getMessageCount); + + Wait.assertFalse(queue.getPagingStore()::isPaging, 1000, 100); + + server.stop(); + + // Deleting the paging data. Simulating a failure + // a dumb user, or anything that will remove the data + deleteDirectory(new File(getPageDir())); + + server = createServer(true, config, JournalPagingTest.PAGE_SIZE, JournalPagingTest.PAGE_MAX); + server.start(); + + locator = createInVMNonHALocator().setBlockOnNonDurableSend(true).setBlockOnDurableSend(true).setBlockOnAcknowledge(true); + + sf = createSessionFactory(locator); + + queue = server.locateQueue(ADDRESS); + + sf = createSessionFactory(locator); + session = sf.createSession(false, false, false); + + producer = session.createProducer(JournalPagingTest.ADDRESS); + + for (int i = 0; i < numberOfMessages * 2; i++) { + message = session.createMessage(true); + + ActiveMQBuffer bodyLocal = message.getBodyBuffer(); + + bodyLocal.writeBytes(body); + + message.putIntProperty(new SimpleString("theid"), i); + + producer.send(message); + if (i % 1000 == 0) { + session.commit(); + } + } + + session.commit(); + + server.stop(); + + server = createServer(true, config, JournalPagingTest.PAGE_SIZE, JournalPagingTest.PAGE_MAX); + server.start(); + + locator = createInVMNonHALocator(); + sf = createSessionFactory(locator); + + sessionConsumer = sf.createSession(false, false, false); + sessionConsumer.start(); + consumer = sessionConsumer.createConsumer(JournalPagingTest.ADDRESS); + for (int msgCount = 0; msgCount < numberOfMessages; msgCount++) { + logger.debug("Received {}", msgCount); + ClientMessage msg = consumer.receive(100); + if (msg == null) { + logger.debug("It's null. leaving now"); + sessionConsumer.commit(); + fail("Didn't receive a message"); + } + msg.acknowledge(); + + if (msgCount % 5 == 0) { + logger.debug("commit"); + sessionConsumer.commit(); + } + } + + sessionConsumer.commit(); + + sessionConsumer.close(); + + } + + @Override + protected void applySettings(ActiveMQServer server, + final Configuration configuration, + final int pageSize, + final long maxAddressSize, + final Integer maxReadPageMessages, + final Integer maxReadPageBytes, + final Map settings) { + server.getConfiguration().setAddressQueueScanPeriod(100); + } +} diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingOrderTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingOrderTest.java index bbc01212dc..613480071c 100644 --- a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingOrderTest.java +++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingOrderTest.java @@ -56,6 +56,7 @@ import org.apache.activemq.artemis.jms.client.ActiveMQJMSConnectionFactory; import org.apache.activemq.artemis.jms.server.impl.JMSServerManagerImpl; import org.apache.activemq.artemis.tests.unit.util.InVMNamingContext; import org.apache.activemq.artemis.tests.util.ActiveMQTestBase; +import org.apache.activemq.artemis.tests.util.Wait; import org.junit.Test; /** @@ -69,7 +70,7 @@ public class PagingOrderTest extends ActiveMQTestBase { private static final int PAGE_SIZE = 10 * 1024; - static final SimpleString ADDRESS = new SimpleString("SimpleAddress"); + static final SimpleString ADDRESS = new SimpleString("TestQueue"); private Connection conn; @@ -96,7 +97,7 @@ public class PagingOrderTest extends ActiveMQTestBase { server.addAddressInfo(new AddressInfo(ADDRESS, RoutingType.ANYCAST)); server.createQueue(new QueueConfiguration(ADDRESS).setRoutingType(RoutingType.ANYCAST)); - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); + ClientProducer producer = session.createProducer(ADDRESS); byte[] body = new byte[messageSize]; @@ -188,7 +189,7 @@ public class PagingOrderTest extends ActiveMQTestBase { Queue q2 = server.createQueue(new QueueConfiguration(new SimpleString("inactive")).setAddress(ADDRESS).setRoutingType(RoutingType.MULTICAST)); - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); + ClientProducer producer = session.createProducer(ADDRESS); byte[] body = new byte[messageSize]; @@ -285,11 +286,15 @@ public class PagingOrderTest extends ActiveMQTestBase { assertNotNull(q2); - assertEquals("q2 msg count", numberOfMessages, getMessageCount(q2)); - assertEquals("q2 msgs added", numberOfMessages, getMessagesAdded(q2)); - assertEquals("q1 msg count", 0, getMessageCount(q1)); - // 0, since nothing was sent to the queue after the server was restarted - assertEquals("q1 msgs added", 0, getMessagesAdded(q1)); + { + Queue finalQ2 = q2; + Queue finalQ1 = q1; + Wait.assertEquals(numberOfMessages, () -> getMessageCount(finalQ2), 5000); + Wait.assertEquals(numberOfMessages, () -> getMessagesAdded(finalQ2), 5000); + Wait.assertEquals(0, () -> getMessageCount(finalQ1)); + // 0, since nothing was sent to the queue after the server was restarted + Wait.assertEquals(0, () -> getMessagesAdded(finalQ1)); + } } @@ -318,7 +323,7 @@ public class PagingOrderTest extends ActiveMQTestBase { Queue q2 = server.createQueue(new QueueConfiguration(new SimpleString("inactive")).setAddress(ADDRESS).setRoutingType(RoutingType.MULTICAST)); - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); + ClientProducer producer = session.createProducer(ADDRESS); byte[] body = new byte[messageSize]; @@ -410,7 +415,7 @@ public class PagingOrderTest extends ActiveMQTestBase { server.addAddressInfo(new AddressInfo(ADDRESS, RoutingType.ANYCAST)); server.createQueue(new QueueConfiguration(ADDRESS).setRoutingType(RoutingType.ANYCAST)); - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); + ClientProducer producer = session.createProducer(ADDRESS); byte[] body = new byte[messageSize]; @@ -495,7 +500,7 @@ public class PagingOrderTest extends ActiveMQTestBase { server.addAddressInfo(new AddressInfo(ADDRESS, RoutingType.ANYCAST)); QueueImpl queue = (QueueImpl) server.createQueue(new QueueConfiguration(ADDRESS).setRoutingType(RoutingType.ANYCAST)); - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); + ClientProducer producer = session.createProducer(ADDRESS); byte[] body = new byte[messageSize]; diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingSyncTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingSyncTest.java index 243c376d1d..cec3dc995e 100644 --- a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingSyncTest.java +++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/paging/PagingSyncTest.java @@ -48,7 +48,7 @@ public class PagingSyncTest extends ActiveMQTestBase { - static final SimpleString ADDRESS = new SimpleString("SimpleAddress"); + static final SimpleString ADDRESS = new SimpleString("TestQueue"); @Test public void testOrder1() throws Throwable { @@ -73,7 +73,7 @@ public class PagingSyncTest extends ActiveMQTestBase { server.addAddressInfo(new AddressInfo(ADDRESS, RoutingType.ANYCAST)); server.createQueue(new QueueConfiguration(ADDRESS).setRoutingType(RoutingType.ANYCAST)); - ClientProducer producer = session.createProducer(PagingTest.ADDRESS); + ClientProducer producer = session.createProducer(ADDRESS); byte[] body = new byte[messageSize]; diff --git a/tests/performance-tests/src/test/java/org/apache/activemq/artemis/tests/performance/storage/PersistMultiThreadTest.java b/tests/performance-tests/src/test/java/org/apache/activemq/artemis/tests/performance/storage/PersistMultiThreadTest.java index 4317ceaea1..46e0065fdc 100644 --- a/tests/performance-tests/src/test/java/org/apache/activemq/artemis/tests/performance/storage/PersistMultiThreadTest.java +++ b/tests/performance-tests/src/test/java/org/apache/activemq/artemis/tests/performance/storage/PersistMultiThreadTest.java @@ -387,7 +387,7 @@ public class PersistMultiThreadTest extends ActiveMQTestBase { } @Override - public void sync() throws Exception { + public void addSyncPoint(OperationContext context) throws Exception { } diff --git a/tests/pom.xml b/tests/pom.xml index 5116436d5c..87417613df 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -141,5 +141,6 @@ smoke-tests e2e-tests leak-tests + db-tests diff --git a/tests/smoke-tests/pom.xml b/tests/smoke-tests/pom.xml index 9a7c4e6887..271de9e7d1 100644 --- a/tests/smoke-tests/pom.xml +++ b/tests/smoke-tests/pom.xml @@ -29,7 +29,7 @@ ${project.basedir}/../../ - -Ddistribution.lib="${activemq.basedir}/artemis-distribution/target/apache-artemis-${project.version}-bin/apache-artemis-${project.version}/lib" + -Ddistribution.lib="${activemq.basedir}/artemis-distribution/target/apache-artemis-${project.version}-bin/apache-artemis-${project.version}/lib" localhost @@ -1553,7 +1553,7 @@ 1 false ${skipSmokeTests} - ${sts-surefire-extra-args} ${activemq-surefire-argline} ${artemis-distribuiton-lib-dir} + ${sts-surefire-extra-args} ${activemq-surefire-argline} ${artemis-distribution-lib-dir} diff --git a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/common/SmokeTestBase.java b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/common/SmokeTestBase.java index 8c97d60905..c9c1262413 100644 --- a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/common/SmokeTestBase.java +++ b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/common/SmokeTestBase.java @@ -16,178 +16,7 @@ */ package org.apache.activemq.artemis.tests.smoke.common; -import javax.management.MBeanServerInvocationHandler; -import javax.management.remote.JMXConnector; -import javax.management.remote.JMXConnectorFactory; -import javax.management.remote.JMXServiceURL; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.HashSet; -import java.util.Set; +import org.apache.activemq.artemis.utils.RealServerTestBase; -import org.apache.activemq.artemis.api.core.management.ActiveMQServerControl; -import org.apache.activemq.artemis.api.core.management.ObjectNameBuilder; -import org.apache.activemq.artemis.cli.commands.Stop; -import org.apache.activemq.artemis.tests.util.ActiveMQTestBase; -import org.apache.activemq.artemis.util.ServerUtil; -import org.junit.After; -import org.junit.Assert; - -public class SmokeTestBase extends ActiveMQTestBase { - Set processes = new HashSet<>(); - private static final String JMX_SERVER_HOSTNAME = "localhost"; - private static final int JMX_SERVER_PORT = 10099; - - public static final String basedir = System.getProperty("basedir"); - - @After - public void after() throws Exception { - // close ServerLocators before killing the server otherwise they'll hang and delay test termination - closeAllServerLocatorsFactories(); - - for (Process process : processes) { - try { - ServerUtil.killServer(process, true); - } catch (Throwable e) { - e.printStackTrace(); - } - } - processes.clear(); - } - - public void killServer(Process process) { - processes.remove(process); - try { - ServerUtil.killServer(process); - } catch (Throwable e) { - e.printStackTrace(); - } - } - - protected static void stopServerWithFile(String serverLocation) throws IOException { - File serverPlace = new File(serverLocation); - File etcPlace = new File(serverPlace, "etc"); - File stopMe = new File(etcPlace, Stop.STOP_FILE_NAME); - Assert.assertTrue(stopMe.createNewFile()); - } - - public static String getServerLocation(String serverName) { - return basedir + "/target/" + serverName; - } - - public static void cleanupData(String serverName) { - String location = getServerLocation(serverName); - deleteDirectory(new File(location, "data")); - deleteDirectory(new File(location, "log")); - } - - public void addProcess(Process process) { - processes.add(process); - } - - public void removeProcess(Process process) { - processes.remove(process); - } - - public Process startServer(String serverName, int portID, int timeout) throws Exception { - Process process = ServerUtil.startServer(getServerLocation(serverName), serverName, portID, timeout); - addProcess(process); - return process; - } - - public Process startServer(String serverName, String uri, int timeout) throws Exception { - Process process = ServerUtil.startServer(getServerLocation(serverName), serverName, uri, timeout); - addProcess(process); - return process; - } - - protected JMXConnector getJmxConnector() throws MalformedURLException { - return getJmxConnector(JMX_SERVER_HOSTNAME, JMX_SERVER_PORT); - } - - protected static JMXConnector newJMXFactory(String uri) throws Throwable { - return JMXConnectorFactory.connect(new JMXServiceURL(uri)); - } - - protected static ActiveMQServerControl getServerControl(String uri, - ObjectNameBuilder builder, - long timeout) throws Throwable { - long expireLoop = System.currentTimeMillis() + timeout; - Throwable lastException = null; - do { - try { - JMXConnector connector = newJMXFactory(uri); - - ActiveMQServerControl serverControl = MBeanServerInvocationHandler.newProxyInstance(connector.getMBeanServerConnection(), builder.getActiveMQServerObjectName(), ActiveMQServerControl.class, false); - serverControl.isActive(); // making one call to make sure it's working - return serverControl; - } catch (Throwable e) { - lastException = e; - Thread.sleep(500); - } - } - while (expireLoop > System.currentTimeMillis()); - - throw lastException; - } - - protected static JMXConnector getJmxConnector(String hostname, int port) throws MalformedURLException { - // Without this, the RMI server would bind to the default interface IP (the user's local IP mostly) - System.setProperty("java.rmi.server.hostname", hostname); - - // I don't specify both ports here manually on purpose. See actual RMI registry connection port extraction below. - String urlString = "service:jmx:rmi:///jndi/rmi://" + hostname + ":" + port + "/jmxrmi"; - - JMXServiceURL url = new JMXServiceURL(urlString); - JMXConnector jmxConnector = null; - - try { - jmxConnector = JMXConnectorFactory.connect(url); - System.out.println("Successfully connected to: " + urlString); - } catch (Exception e) { - jmxConnector = null; - e.printStackTrace(); - Assert.fail(e.getMessage()); - } - return jmxConnector; - } - - protected static final void recreateBrokerDirectory(final String homeInstance) { - recreateDirectory(homeInstance + "/data"); - recreateDirectory(homeInstance + "/log"); - } - - protected void checkLogRecord(File logFile, boolean exist, String... values) throws Exception { - Assert.assertTrue(logFile.exists()); - boolean hasRecord = false; - try (BufferedReader reader = new BufferedReader(new FileReader(logFile))) { - String line = reader.readLine(); - while (line != null) { - if (line.contains(values[0])) { - boolean hasAll = true; - for (int i = 1; i < values.length; i++) { - if (!line.contains(values[i])) { - hasAll = false; - break; - } - } - if (hasAll) { - hasRecord = true; - System.out.println("audit has it: " + line); - break; - } - } - line = reader.readLine(); - } - if (exist) { - Assert.assertTrue(hasRecord); - } else { - Assert.assertFalse(hasRecord); - } - } - } - -} +public class SmokeTestBase extends RealServerTestBase { +} \ No newline at end of file diff --git a/tests/soak-tests/pom.xml b/tests/soak-tests/pom.xml index 9995f00b59..dbd48d3f26 100644 --- a/tests/soak-tests/pom.xml +++ b/tests/soak-tests/pom.xml @@ -581,83 +581,6 @@ - - test-compile - create-database-paging - - create - - - - -Djava.net.preferIPv4Stack=true - ${basedir}/target/database-paging/derby - ${basedir}/target/classes/servers/database-paging/derby - - org.apache.derby:derby:${apache.derby.version} - - - --jdbc - --global-max-messages - 100 - --java-options - -ea - - - - - test-compile - create-database-paging-mysql - - create - - - - -Djava.net.preferIPv4Stack=true - ${basedir}/target/database-paging/mysql - ${basedir}/target/classes/servers/database-paging/mysql - - com.mysql:mysql-connector-j:8.0.33 - - - --jdbc - --jdbc-connection-url - jdbc:mysql://localhost/ARTEMIS-TEST?user=root&#38;password=artemis - --jdbc-driver-class-name - com.mysql.cj.jdbc.Driver - --global-max-messages - 100 - --java-options - -ea - - - - - test-compile - create-database-paging-postgres - - create - - - - -Djava.net.preferIPv4Stack=true - ${basedir}/target/database-paging/postgres - ${basedir}/target/classes/servers/database-paging/postgres - - org.postgresql:postgresql:42.6.0 - - - --jdbc - --jdbc-connection-url - jdbc:postgresql:artemis?user=artemis&#38;password=artemis - --jdbc-driver-class-name - org.postgresql.Driver - --global-max-messages - 100 - --java-options - -ea - - - test-compile create-paging-export diff --git a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/SoakTestBase.java b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/SoakTestBase.java index 02dd357eff..687b5deb62 100644 --- a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/SoakTestBase.java +++ b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/SoakTestBase.java @@ -16,246 +16,14 @@ */ package org.apache.activemq.artemis.tests.soak; -import javax.management.MBeanServerInvocationHandler; -import javax.management.ObjectName; -import javax.management.remote.JMXConnector; -import javax.management.remote.JMXConnectorFactory; -import javax.management.remote.JMXServiceURL; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.HashSet; -import java.util.Set; - -import org.apache.activemq.artemis.api.core.RoutingType; -import org.apache.activemq.artemis.api.core.SimpleString; -import org.apache.activemq.artemis.api.core.management.ActiveMQServerControl; -import org.apache.activemq.artemis.api.core.management.ObjectNameBuilder; -import org.apache.activemq.artemis.api.core.management.QueueControl; -import org.apache.activemq.artemis.api.jms.ActiveMQJMSClient; -import org.apache.activemq.artemis.cli.commands.Stop; -import org.apache.activemq.artemis.jms.client.ActiveMQConnectionFactory; -import org.apache.activemq.artemis.tests.util.ActiveMQTestBase; -import org.apache.activemq.artemis.util.ServerUtil; -import org.apache.activemq.artemis.utils.SpawnedVMSupport; -import org.junit.After; -import org.junit.Assert; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.lang.invoke.MethodHandles; -public class SoakTestBase extends ActiveMQTestBase { +import org.apache.activemq.artemis.utils.RealServerTestBase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SoakTestBase extends RealServerTestBase { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - Set processes = new HashSet<>(); - private static final String JMX_SERVER_HOSTNAME = "localhost"; - private static final int JMX_SERVER_PORT = 10099; - - public static final String basedir = System.getProperty("basedir"); - - protected static void unzip(File zipFile, File serverFolder) throws IOException, ClassNotFoundException, InterruptedException { - ProcessBuilder zipBuilder = new ProcessBuilder("unzip", zipFile.getAbsolutePath()).directory(serverFolder); - - Process process = zipBuilder.start(); - SpawnedVMSupport.startLogger("zip", process); - logger.info("Zip finished with {}", process.waitFor()); - } - - - protected static void zip(File zipFile, File serverFolder) throws IOException, ClassNotFoundException, InterruptedException { - logger.info("Zipping data folder for {}", zipFile); - ProcessBuilder zipBuilder = new ProcessBuilder("zip", "-r", zipFile.getAbsolutePath(), "data").directory(serverFolder); - Process process = zipBuilder.start(); - SpawnedVMSupport.startLogger("zip", process); - logger.info("Zip finished with {}", process.waitFor()); - } - - - @After - public void after() throws Exception { - for (Process process : processes) { - try { - ServerUtil.killServer(process, true); - } catch (Throwable e) { - e.printStackTrace(); - } - } - processes.clear(); - } - - public void killServer(Process process) { - processes.remove(process); - try { - ServerUtil.killServer(process); - } catch (Throwable e) { - e.printStackTrace(); - } - } - - protected static void stopServerWithFile(String serverLocation) throws IOException { - File serverPlace = new File(serverLocation); - File etcPlace = new File(serverPlace, "etc"); - File stopMe = new File(etcPlace, Stop.STOP_FILE_NAME); - Assert.assertTrue(stopMe.createNewFile()); - } - - public static String getServerLocation(String serverName) { - return basedir + "/target/" + serverName; - } - - public static void cleanupData(String serverName) { - String location = getServerLocation(serverName); - deleteDirectory(new File(location, "data")); - deleteDirectory(new File(location, "log")); - } - - public void addProcess(Process process) { - processes.add(process); - } - - public Process startServer(String serverName, int portID, int timeout) throws Exception { - Process process = ServerUtil.startServer(getServerLocation(serverName), serverName, portID, timeout); - addProcess(process); - return process; - } - - public Process startServer(String serverName, String uri, int timeout) throws Exception { - Process process = ServerUtil.startServer(getServerLocation(serverName), serverName, uri, timeout); - addProcess(process); - return process; - } - - protected JMXConnector getJmxConnector() throws MalformedURLException { - return getJmxConnector(JMX_SERVER_HOSTNAME, JMX_SERVER_PORT); - } - - protected static JMXConnector newJMXFactory(String uri) throws Throwable { - return JMXConnectorFactory.connect(new JMXServiceURL(uri)); - } - - protected static ActiveMQServerControl getServerControl(String uri, - ObjectNameBuilder builder, - long timeout) throws Throwable { - long expireLoop = System.currentTimeMillis() + timeout; - Throwable lastException = null; - do { - try { - JMXConnector connector = newJMXFactory(uri); - - ActiveMQServerControl serverControl = MBeanServerInvocationHandler.newProxyInstance(connector.getMBeanServerConnection(), builder.getActiveMQServerObjectName(), ActiveMQServerControl.class, false); - serverControl.isActive(); // making one call to make sure it's working - return serverControl; - } catch (Throwable e) { - lastException = e; - Thread.sleep(500); - } - } - while (expireLoop > System.currentTimeMillis()); - - throw lastException; - } - - protected static QueueControl getQueueControl(String uri, - ObjectNameBuilder builder, - String address, - String queueName, - RoutingType routingType, - long timeout) throws Throwable { - long expireLoop = System.currentTimeMillis() + timeout; - Throwable lastException = null; - do { - try { - JMXConnector connector = newJMXFactory(uri); - - ObjectName objectQueueName = builder.getQueueObjectName(SimpleString.toSimpleString(address), SimpleString.toSimpleString(queueName), routingType); - - QueueControl queueControl = MBeanServerInvocationHandler.newProxyInstance(connector.getMBeanServerConnection(), objectQueueName, QueueControl.class, false); - queueControl.getMessagesAcknowledged(); // making one call - return queueControl; - } catch (Throwable e) { - logger.warn(e.getMessage(), e); - lastException = e; - Thread.sleep(500); - } - } - while (expireLoop > System.currentTimeMillis()); - - throw lastException; - } - - protected static JMXConnector getJmxConnector(String hostname, int port) throws MalformedURLException { - // Without this, the RMI server would bind to the default interface IP (the user's local IP mostly) - System.setProperty("java.rmi.server.hostname", hostname); - - // I don't specify both ports here manually on purpose. See actual RMI registry connection port extraction below. - String urlString = "service:jmx:rmi:///jndi/rmi://" + hostname + ":" + port + "/jmxrmi"; - - JMXServiceURL url = new JMXServiceURL(urlString); - JMXConnector jmxConnector = null; - - try { - jmxConnector = JMXConnectorFactory.connect(url); - System.out.println("Successfully connected to: " + urlString); - } catch (Exception e) { - jmxConnector = null; - e.printStackTrace(); - Assert.fail(e.getMessage()); - } - return jmxConnector; - } - - protected static final void recreateBrokerDirectory(final String homeInstance) { - recreateDirectory(homeInstance + "/data"); - recreateDirectory(homeInstance + "/log"); - } - - - public boolean waitForServerToStart(String uri, String username, String password, long timeout) throws InterruptedException { - long realTimeout = System.currentTimeMillis() + timeout; - while (System.currentTimeMillis() < realTimeout) { - try (ActiveMQConnectionFactory cf = ActiveMQJMSClient.createConnectionFactory(uri, null)) { - cf.createConnection(username, password).close(); - System.out.println("server " + uri + " started"); - } catch (Exception e) { - System.out.println("awaiting server " + uri + " start at "); - Thread.sleep(500); - continue; - } - return true; - } - - return false; - } - - protected void checkLogRecord(File logFile, boolean exist, String... values) throws Exception { - Assert.assertTrue(logFile.exists()); - boolean hasRecord = false; - try (BufferedReader reader = new BufferedReader(new FileReader(logFile))) { - String line = reader.readLine(); - while (line != null) { - if (line.contains(values[0])) { - boolean hasAll = true; - for (int i = 1; i < values.length; i++) { - if (!line.contains(values[i])) { - hasAll = false; - break; - } - } - if (hasAll) { - hasRecord = true; - System.out.println("audit has it: " + line); - break; - } - } - line = reader.readLine(); - } - if (exist) { - Assert.assertTrue(hasRecord); - } else { - Assert.assertFalse(hasRecord); - } - } - } } diff --git a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/client/ClientParameters.java b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/client/ClientParameters.java index 5f2ac8e4fd..d3fa9a536a 100644 --- a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/client/ClientParameters.java +++ b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/client/ClientParameters.java @@ -17,7 +17,7 @@ package org.apache.activemq.artemis.tests.soak.client; -import static org.apache.activemq.artemis.tests.soak.TestParameters.testProperty; +import static org.apache.activemq.artemis.utils.TestParameters.testProperty; public class ClientParameters { private static final String TEST_NAME = "CLIENT"; diff --git a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/failover/RandomFailoverSoakTest.java b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/failover/RandomFailoverSoakTest.java index 20c69da434..b37ca0aadc 100644 --- a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/failover/RandomFailoverSoakTest.java +++ b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/failover/RandomFailoverSoakTest.java @@ -18,7 +18,7 @@ package org.apache.activemq.artemis.tests.soak.failover; import org.apache.activemq.artemis.tests.integration.cluster.reattach.RandomReattachTest; -import static org.apache.activemq.artemis.tests.soak.TestParameters.testProperty; +import static org.apache.activemq.artemis.utils.TestParameters.testProperty; public class RandomFailoverSoakTest extends RandomReattachTest { diff --git a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/owleak/OWLeakTest.java b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/owleak/OWLeakTest.java index 944ee54928..bcaa32896b 100644 --- a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/owleak/OWLeakTest.java +++ b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/owleak/OWLeakTest.java @@ -47,8 +47,8 @@ import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.activemq.artemis.tests.soak.TestParameters.intMandatoryProperty; -import static org.apache.activemq.artemis.tests.soak.TestParameters.testProperty; +import static org.apache.activemq.artemis.utils.TestParameters.intMandatoryProperty; +import static org.apache.activemq.artemis.utils.TestParameters.testProperty; /** * Refer to ./scripts/parameters.sh for suggested parameters diff --git a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/FlowControlPagingTest.java b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/FlowControlPagingTest.java index 821ad04d5b..f5bdaa0306 100644 --- a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/FlowControlPagingTest.java +++ b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/FlowControlPagingTest.java @@ -44,8 +44,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.invoke.MethodHandles; -import static org.apache.activemq.artemis.tests.soak.TestParameters.intMandatoryProperty; -import static org.apache.activemq.artemis.tests.soak.TestParameters.testProperty; +import static org.apache.activemq.artemis.utils.TestParameters.intMandatoryProperty; +import static org.apache.activemq.artemis.utils.TestParameters.testProperty; /** * Refer to ./scripts/parameters.sh for suggested parameters diff --git a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/HorizontalPagingTest.java b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/HorizontalPagingTest.java index 457deff0e0..16c9f262f1 100644 --- a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/HorizontalPagingTest.java +++ b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/HorizontalPagingTest.java @@ -45,8 +45,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.invoke.MethodHandles; -import static org.apache.activemq.artemis.tests.soak.TestParameters.intMandatoryProperty; -import static org.apache.activemq.artemis.tests.soak.TestParameters.testProperty; +import static org.apache.activemq.artemis.utils.TestParameters.intMandatoryProperty; +import static org.apache.activemq.artemis.utils.TestParameters.testProperty; /** * Refer to ./scripts/parameters.sh for suggested parameters diff --git a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/SubscriptionPagingTest.java b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/SubscriptionPagingTest.java index 297a229938..0cd41043e6 100644 --- a/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/SubscriptionPagingTest.java +++ b/tests/soak-tests/src/test/java/org/apache/activemq/artemis/tests/soak/paging/SubscriptionPagingTest.java @@ -47,8 +47,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.invoke.MethodHandles; -import static org.apache.activemq.artemis.tests.soak.TestParameters.intMandatoryProperty; -import static org.apache.activemq.artemis.tests.soak.TestParameters.testProperty; +import static org.apache.activemq.artemis.utils.TestParameters.intMandatoryProperty; +import static org.apache.activemq.artemis.utils.TestParameters.testProperty; /** * Refer to ./scripts/parameters.sh for suggested parameters diff --git a/tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/paging/impl/PagingStoreImplTest.java b/tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/paging/impl/PagingStoreImplTest.java index 2ebba5dba5..013f241cce 100644 --- a/tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/paging/impl/PagingStoreImplTest.java +++ b/tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/paging/impl/PagingStoreImplTest.java @@ -56,6 +56,7 @@ import org.apache.activemq.artemis.core.paging.impl.PageTransactionInfoImpl; import org.apache.activemq.artemis.core.paging.impl.PagingStoreImpl; import org.apache.activemq.artemis.core.paging.impl.PagingStoreTestAccessor; import org.apache.activemq.artemis.core.persistence.StorageManager; +import org.apache.activemq.artemis.core.persistence.impl.journal.OperationContextImpl; import org.apache.activemq.artemis.core.persistence.impl.nullpm.NullStorageManager; import org.apache.activemq.artemis.core.server.files.FileStoreMonitor; import org.apache.activemq.artemis.core.server.impl.RoutingContextImpl; @@ -185,7 +186,7 @@ public class PagingStoreImplTest extends ActiveMQTestBase { Assert.assertEquals(1, storeImpl.getNumberOfPages()); - storeImpl.sync(); + storeImpl.addSyncPoint(OperationContextImpl.getContext()); storeImpl = new PagingStoreImpl(PagingStoreImplTest.destinationTestName, null, 100, createMockManager(), createStorageManagerMock(), factory, storeFactory, PagingStoreImplTest.destinationTestName, addressSettings, getExecutorFactory().getExecutor(), getExecutorFactory().getExecutor(), true); @@ -230,7 +231,7 @@ public class PagingStoreImplTest extends ActiveMQTestBase { Assert.assertEquals(1, storeImpl.getNumberOfPages()); - storeImpl.sync(); + storeImpl.addSyncPoint(OperationContextImpl.getContext()); Page page = storeImpl.depage(); @@ -567,7 +568,7 @@ public class PagingStoreImplTest extends ActiveMQTestBase { Assert.assertEquals(2, store.getNumberOfPages()); - store.sync(); + store.addSyncPoint(OperationContextImpl.getContext()); int sequence = 0;