diff --git a/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/Create.java b/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/Create.java index bd0b4cd2e2..2897f6d93a 100644 --- a/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/Create.java +++ b/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/Create.java @@ -38,6 +38,7 @@ import java.util.regex.Pattern; import io.airlift.airline.Arguments; import io.airlift.airline.Command; import io.airlift.airline.Option; +import org.apache.activemq.artemis.api.config.ActiveMQDefaultConfiguration; import org.apache.activemq.artemis.cli.CLIException; import org.apache.activemq.artemis.cli.commands.util.HashUtil; import org.apache.activemq.artemis.cli.commands.util.SyncCalculation; @@ -723,6 +724,16 @@ public class Create extends InputAbstract { } private void setupJournalType() { + + if (noJournalSync && !mapped) { + boolean useMapped = inputBoolean("--mapped", "Since you disabled syncs, it is recommended to use the Mapped Memory Journal. Do you want to use the Memory Mapped Journal", true); + + if (useMapped) { + mapped = true; + nio = false; + aio = false; + } + } int countJournalTypes = countBoolean(aio, nio, mapped); if (countJournalTypes > 1) { throw new RuntimeException("You can only select one journal type (--nio | --aio | --mapped)."); @@ -803,20 +814,34 @@ public class Create extends InputAbstract { System.out.println(""); System.out.println("Auto tuning journal ..."); - long time = SyncCalculation.syncTest(dataFolder, 4096, writes, 5, verbose, !noJournalSync, journalType); - long nanoseconds = SyncCalculation.toNanos(time, writes, verbose); - double writesPerMillisecond = (double) writes / (double) time; + if (mapped && noJournalSync) { + HashMap syncFilter = new HashMap<>(); + syncFilter.put("${nanoseconds}", "0"); + syncFilter.put("${writesPerMillisecond}", "0"); + syncFilter.put("${maxaio}", journalType == JournalType.ASYNCIO ? "" + ActiveMQDefaultConfiguration.getDefaultJournalMaxIoAio() : "1"); - String writesPerMillisecondStr = new DecimalFormat("###.##").format(writesPerMillisecond); + System.out.println("...Since you disabled sync and are using MAPPED journal, we are diabling buffer times"); - HashMap syncFilter = new HashMap<>(); - syncFilter.put("${nanoseconds}", Long.toString(nanoseconds)); - syncFilter.put("${writesPerMillisecond}", writesPerMillisecondStr); + filters.put("${journal-buffer.settings}", readTextFile(ETC_JOURNAL_BUFFER_SETTINGS, syncFilter)); - System.out.println("done! Your system can make " + writesPerMillisecondStr + - " writes per millisecond, your journal-buffer-timeout will be " + nanoseconds); + } else { + long time = SyncCalculation.syncTest(dataFolder, 4096, writes, 5, verbose, !noJournalSync, false, "journal-test.tmp", ActiveMQDefaultConfiguration.getDefaultJournalMaxIoAio(), journalType); + long nanoseconds = SyncCalculation.toNanos(time, writes, verbose); + double writesPerMillisecond = (double) writes / (double) time; + + String writesPerMillisecondStr = new DecimalFormat("###.##").format(writesPerMillisecond); + + HashMap syncFilter = new HashMap<>(); + syncFilter.put("${nanoseconds}", Long.toString(nanoseconds)); + syncFilter.put("${writesPerMillisecond}", writesPerMillisecondStr); + syncFilter.put("${maxaio}", journalType == JournalType.ASYNCIO ? "" + ActiveMQDefaultConfiguration.getDefaultJournalMaxIoAio() : "1"); + + System.out.println("done! Your system can make " + writesPerMillisecondStr + + " writes per millisecond, your journal-buffer-timeout will be " + nanoseconds); + + filters.put("${journal-buffer.settings}", readTextFile(ETC_JOURNAL_BUFFER_SETTINGS, syncFilter)); + } - filters.put("${journal-buffer.settings}", readTextFile(ETC_JOURNAL_BUFFER_SETTINGS, syncFilter)); } catch (Exception e) { filters.put("${journal-buffer.settings}", ""); diff --git a/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/tools/journal/PerfJournal.java b/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/tools/journal/PerfJournal.java index 3805de67d7..ab2769f982 100644 --- a/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/tools/journal/PerfJournal.java +++ b/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/tools/journal/PerfJournal.java @@ -22,13 +22,13 @@ import java.text.DecimalFormat; import io.airlift.airline.Command; import io.airlift.airline.Option; import org.apache.activemq.artemis.cli.commands.ActionContext; -import org.apache.activemq.artemis.cli.commands.tools.LockAbstract; +import org.apache.activemq.artemis.cli.commands.tools.OptionalLocking; import org.apache.activemq.artemis.cli.commands.util.SyncCalculation; import org.apache.activemq.artemis.core.config.impl.FileConfiguration; import org.apache.activemq.artemis.core.server.JournalType; @Command(name = "perf-journal", description = "Calculates the journal-buffer-timeout you should use with the current data folder") -public class PerfJournal extends LockAbstract { +public class PerfJournal extends OptionalLocking { @Option(name = "--block-size", description = "The block size for each write (default 4096)") @@ -49,6 +49,15 @@ public class PerfJournal extends LockAbstract { @Option(name = "--journal-type", description = "Journal Type to be used (default from broker.xml)") public String journalType = null; + @Option(name = "--sync-writes", description = "It will perform each write synchronously, like if you had a single producer") + public boolean syncWrites = false; + + @Option(name = "--file", description = "The file name to be used (default test.tmp)") + public String fileName = "test.tmp"; + + @Option(name = "--max-aio", description = "libaio.maxAIO to be used (default: configuration::getJournalMaxIO_AIO()") + public int maxAIO = 0; + @Override public Object execute(ActionContext context) throws Exception { @@ -74,7 +83,11 @@ public class PerfJournal extends LockAbstract { fileConfiguration.getJournalLocation().mkdirs(); - long time = SyncCalculation.syncTest(fileConfiguration.getJournalLocation(), size, writes, tries, verbose, fileConfiguration.isJournalDatasync(), fileConfiguration.getJournalType()); + if (maxAIO <= 0) { + maxAIO = fileConfiguration.getJournalMaxIO_AIO(); + } + + long time = SyncCalculation.syncTest(fileConfiguration.getJournalLocation(), size, writes, tries, verbose, fileConfiguration.isJournalDatasync(), syncWrites, fileName, maxAIO, fileConfiguration.getJournalType()); long nanosecondsWait = SyncCalculation.toNanos(time, writes, verbose); double writesPerMillisecond = (double) writes / (double) time; diff --git a/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/util/SyncCalculation.java b/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/util/SyncCalculation.java index 860bcb6818..5cc6ab994b 100644 --- a/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/util/SyncCalculation.java +++ b/artemis-cli/src/main/java/org/apache/activemq/artemis/cli/commands/util/SyncCalculation.java @@ -50,13 +50,16 @@ public class SyncCalculation { int tries, boolean verbose, boolean fsync, + boolean syncWrites, + String fileName, + int maxAIO, JournalType journalType) throws Exception { - SequentialFileFactory factory = newFactory(datafolder, fsync, journalType, blockSize * blocks); + SequentialFileFactory factory = newFactory(datafolder, fsync, journalType, blockSize * blocks, maxAIO); if (verbose) { - System.out.println("Using " + factory.getClass().getName() + " to calculate sync times"); + System.out.println("Using " + factory.getClass().getName() + " to calculate sync times, alignment=" + factory.getAlignment()); } - SequentialFile file = factory.createSequentialFile("test.tmp"); + SequentialFile file = factory.createSequentialFile(fileName); try { file.delete(); @@ -106,10 +109,14 @@ public class SyncCalculation { bufferBlock.position(0); latch.countUp(); file.writeDirect(bufferBlock, true, callback); - if (!latch.await(5, TimeUnit.SECONDS)) { - throw new IOException("Callback wasn't called"); + + if (syncWrites) { + flushLatch(latch); } } + + if (!syncWrites) flushLatch(latch); + long end = System.currentTimeMillis(); result[ntry] = (end - start); @@ -150,6 +157,12 @@ public class SyncCalculation { } } + private static void flushLatch(ReusableLatch latch) throws InterruptedException, IOException { + if (!latch.await(5, TimeUnit.SECONDS)) { + throw new IOException("Timed out on receiving IO callback"); + } + } + public static long toNanos(long time, long blocks, boolean verbose) { double blocksPerMillisecond = (double) blocks / (double) (time); @@ -169,7 +182,7 @@ public class SyncCalculation { return timeWait; } - private static SequentialFileFactory newFactory(File datafolder, boolean datasync, JournalType journalType, int fileSize) { + private static SequentialFileFactory newFactory(File datafolder, boolean datasync, JournalType journalType, int fileSize, int maxAIO) { SequentialFileFactory factory; if (journalType == JournalType.ASYNCIO && !LibaioContext.isLoaded()) { @@ -184,12 +197,12 @@ public class SyncCalculation { factory.start(); return factory; case ASYNCIO: - factory = new AIOSequentialFileFactory(datafolder, 1).setDatasync(datasync); + factory = new AIOSequentialFileFactory(datafolder, maxAIO).setDatasync(datasync); factory.start(); ((AIOSequentialFileFactory) factory).disableBufferReuse(); return factory; case MAPPED: - factory = MappedSequentialFileFactory.unbuffered(datafolder, fileSize, null) + factory = new MappedSequentialFileFactory(datafolder, fileSize, false, 0, 0, null) .setDatasync(datasync) .disableBufferReuse(); factory.start(); diff --git a/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/broker.xml b/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/broker.xml index 241c35486a..013b911487 100644 --- a/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/broker.xml +++ b/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/broker.xml @@ -50,7 +50,8 @@ under the License. -1 -${ping-config.settings}${journal-buffer.settings}${connector-config.settings} + 10M + ${journal-buffer.settings}${ping-config.settings}${connector-config.settings} 5000 diff --git a/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/journal-buffer-settings.txt b/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/journal-buffer-settings.txt index 566c29ea78..fc9e2ba1fe 100644 --- a/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/journal-buffer-settings.txt +++ b/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/journal-buffer-settings.txt @@ -3,6 +3,15 @@ This value was determined through a calculation. Your system could perform ${writesPerMillisecond} writes per millisecond on the current journal configuration. - That translates as a sync write every ${nanoseconds} nanoseconds + That translates as a sync write every ${nanoseconds} nanoseconds. + + Note: If you specify 0 the system will perform writes directly to the disk. + We recommend this to be 0 if you are using journalType=MAPPED and ournal-datasync=false. --> ${nanoseconds} + + + + ${maxaio} diff --git a/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/ping-settings.txt b/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/ping-settings.txt index 242c83559e..c7f824d8e9 100644 --- a/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/ping-settings.txt +++ b/artemis-cli/src/main/resources/org/apache/activemq/artemis/cli/commands/etc/ping-settings.txt @@ -1,3 +1,4 @@ +