diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java index 8336543dee8..ea10f602151 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase; import java.io.IOException; +import java.security.SecureRandom; import java.util.Random; import org.apache.commons.logging.Log; @@ -30,6 +31,10 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.io.crypto.Encryption; +import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; +import org.apache.hadoop.hbase.io.crypto.aes.AES; +import org.apache.hadoop.hbase.io.hfile.AbstractHFileWriter; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; @@ -45,7 +50,16 @@ public class HFilePerformanceEvaluation { private static final int ROW_LENGTH = 10; private static final int ROW_COUNT = 1000000; private static final int RFILE_BLOCKSIZE = 8 * 1024; - + private static StringBuilder testSummary = new StringBuilder(); + + // Disable verbose INFO logging from org.apache.hadoop.io.compress.CodecPool + static { + System.setProperty("org.apache.commons.logging.Log", + "org.apache.commons.logging.impl.SimpleLog"); + System.setProperty("org.apache.commons.logging.simplelog.log.org.apache.hadoop.io.compress.CodecPool", + "WARN"); + } + static final Log LOG = LogFactory.getLog(HFilePerformanceEvaluation.class.getName()); @@ -82,70 +96,154 @@ public class HFilePerformanceEvaluation { return CellUtil.createCell(keyRow, value); } + /** + * Add any supported codec or cipher to test the HFile read/write performance. + * Specify "none" to disable codec or cipher or both. + * @throws Exception + */ private void runBenchmarks() throws Exception { final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.get(conf); final Path mf = fs.makeQualified(new Path("performanceevaluation.mapfile")); + + // codec=none cipher=none + runWriteBenchmark(conf, fs, mf, "none", "none"); + runReadBenchmark(conf, fs, mf, "none", "none"); + + // codec=gz cipher=none + runWriteBenchmark(conf, fs, mf, "gz", "none"); + runReadBenchmark(conf, fs, mf, "gz", "none"); + + // Add configuration for AES cipher + final Configuration aesconf = new Configuration(); + aesconf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); + aesconf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); + aesconf.setInt("hfile.format.version", 3); + final FileSystem aesfs = FileSystem.get(aesconf); + final Path aesmf = aesfs.makeQualified(new Path("performanceevaluation.aes.mapfile")); + + // codec=none cipher=aes + runWriteBenchmark(aesconf, aesfs, aesmf, "none", "aes"); + runReadBenchmark(aesconf, aesfs, aesmf, "none", "aes"); + + // codec=gz cipher=aes + runWriteBenchmark(aesconf, aesfs, aesmf, "gz", "aes"); + runReadBenchmark(aesconf, aesfs, aesmf, "gz", "aes"); + + // cleanup test files + if (fs.exists(mf)) { + fs.delete(mf, true); + } + if (aesfs.exists(aesmf)) { + aesfs.delete(aesmf, true); + } + + // Print Result Summary + LOG.info("\n***************\n" + "Result Summary" + "\n***************\n"); + LOG.info(testSummary.toString()); + + } + + /** + * Write a test HFile with the given codec & cipher + * @param conf + * @param fs + * @param mf + * @param codec "none", "lzo", "gz", "snappy" + * @param cipher "none", "aes" + * @throws Exception + */ + private void runWriteBenchmark(Configuration conf, FileSystem fs, Path mf, String codec, + String cipher) throws Exception { if (fs.exists(mf)) { fs.delete(mf, true); } - runBenchmark(new SequentialWriteBenchmark(conf, fs, mf, ROW_COUNT), - ROW_COUNT); + runBenchmark(new SequentialWriteBenchmark(conf, fs, mf, ROW_COUNT, codec, cipher), + ROW_COUNT, codec, cipher); + + } + + /** + * Run all the read benchmarks for the test HFile + * @param conf + * @param fs + * @param mf + * @param codec "none", "lzo", "gz", "snappy" + * @param cipher "none", "aes" + */ + private void runReadBenchmark(final Configuration conf, final FileSystem fs, final Path mf, + final String codec, final String cipher) { PerformanceEvaluationCommons.concurrentReads(new Runnable() { @Override public void run() { try { runBenchmark(new UniformRandomSmallScan(conf, fs, mf, ROW_COUNT), - ROW_COUNT); + ROW_COUNT, codec, cipher); } catch (Exception e) { + testSummary.append("UniformRandomSmallScan failed " + e.getMessage()); e.printStackTrace(); } } }); + PerformanceEvaluationCommons.concurrentReads(new Runnable() { @Override public void run() { try { runBenchmark(new UniformRandomReadBenchmark(conf, fs, mf, ROW_COUNT), - ROW_COUNT); + ROW_COUNT, codec, cipher); } catch (Exception e) { + testSummary.append("UniformRandomReadBenchmark failed " + e.getMessage()); e.printStackTrace(); } } }); + PerformanceEvaluationCommons.concurrentReads(new Runnable() { @Override public void run() { try { runBenchmark(new GaussianRandomReadBenchmark(conf, fs, mf, ROW_COUNT), - ROW_COUNT); + ROW_COUNT, codec, cipher); } catch (Exception e) { + testSummary.append("GaussianRandomReadBenchmark failed " + e.getMessage()); e.printStackTrace(); } } }); + PerformanceEvaluationCommons.concurrentReads(new Runnable() { @Override public void run() { try { runBenchmark(new SequentialReadBenchmark(conf, fs, mf, ROW_COUNT), - ROW_COUNT); + ROW_COUNT, codec, cipher); } catch (Exception e) { + testSummary.append("SequentialReadBenchmark failed " + e.getMessage()); e.printStackTrace(); } } - }); + }); } - - protected void runBenchmark(RowOrientedBenchmark benchmark, int rowCount) - throws Exception { - LOG.info("Running " + benchmark.getClass().getSimpleName() + " for " + - rowCount + " rows."); + + protected void runBenchmark(RowOrientedBenchmark benchmark, int rowCount, + String codec, String cipher) throws Exception { + LOG.info("Running " + benchmark.getClass().getSimpleName() + " with codec[" + + codec + "] " + "cipher[" + cipher + "] for " + rowCount + " rows."); + long elapsedTime = benchmark.run(); - LOG.info("Running " + benchmark.getClass().getSimpleName() + " for " + - rowCount + " rows took " + elapsedTime + "ms."); + + LOG.info("Running " + benchmark.getClass().getSimpleName() + " with codec[" + + codec + "] " + "cipher[" + cipher + "] for " + rowCount + " rows took " + + elapsedTime + "ms."); + + // Store results to print summary at the end + testSummary.append("Running ").append(benchmark.getClass().getSimpleName()) + .append(" with codec[").append(codec).append("] cipher[").append(cipher) + .append("] for ").append(rowCount).append(" rows took ").append(elapsedTime) + .append("ms.").append("\n"); } static abstract class RowOrientedBenchmark { @@ -154,6 +252,18 @@ public class HFilePerformanceEvaluation { protected final FileSystem fs; protected final Path mf; protected final int totalRows; + protected String codec = "none"; + protected String cipher = "none"; + + public RowOrientedBenchmark(Configuration conf, FileSystem fs, Path mf, + int totalRows, String codec, String cipher) { + this.conf = conf; + this.fs = fs; + this.mf = mf; + this.totalRows = totalRows; + this.codec = codec; + this.cipher = cipher; + } public RowOrientedBenchmark(Configuration conf, FileSystem fs, Path mf, int totalRows) { @@ -208,21 +318,36 @@ public class HFilePerformanceEvaluation { private byte[] bytes = new byte[ROW_LENGTH]; public SequentialWriteBenchmark(Configuration conf, FileSystem fs, Path mf, - int totalRows) { - super(conf, fs, mf, totalRows); + int totalRows, String codec, String cipher) { + super(conf, fs, mf, totalRows, codec, cipher); } @Override void setUp() throws Exception { - HFileContext hFileContext = new HFileContextBuilder().withBlockSize(RFILE_BLOCKSIZE).build(); - writer = - HFile.getWriterFactoryNoCache(conf) - .withPath(fs, mf) - .withFileContext(hFileContext) - .withComparator(new KeyValue.RawBytesComparator()) - .create(); - } + HFileContextBuilder builder = new HFileContextBuilder() + .withCompression(AbstractHFileWriter.compressionByName(codec)) + .withBlockSize(RFILE_BLOCKSIZE); + + if (cipher == "aes") { + byte[] cipherKey = new byte[AES.KEY_LENGTH]; + new SecureRandom().nextBytes(cipherKey); + builder.withEncryptionContext(Encryption.newContext(conf) + .setCipher(Encryption.getCipher(conf, cipher)) + .setKey(cipherKey)); + } else if (!"none".equals(cipher)) { + throw new IOException("Cipher " + cipher + " not supported."); + } + + HFileContext hFileContext = builder.build(); + + writer = HFile.getWriterFactoryNoCache(conf) + .withPath(fs, mf) + .withFileContext(hFileContext) + .withComparator(new KeyValue.RawBytesComparator()) + .create(); + } + @Override void doRow(int i) throws Exception { writer.append(createCell(i, generateValue())); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java deleted file mode 100644 index 2bb708514ad..00000000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java +++ /dev/null @@ -1,455 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.io.hfile; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.security.SecureRandom; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Random; - -import org.apache.commons.cli.CommandLine; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.io.crypto.Encryption; -import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; -import org.apache.hadoop.hbase.io.crypto.aes.AES; -import org.apache.hadoop.hbase.util.AbstractHBaseTool; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.compress.CompressionCodec; -import org.apache.hadoop.io.compress.GzipCodec; -import org.apache.hadoop.util.ToolRunner; - -/** - * Set of long-running tests to measure performance of HFile. - *

- * Copied from - * hadoop-3315 tfile. - * Remove after tfile is committed and use the tfile version of this class - * instead.

- */ -public class TestHFilePerformance extends AbstractHBaseTool { - private HBaseTestingUtility TEST_UTIL; - private static String ROOT_DIR; - private FileSystem fs; - private long startTimeEpoch; - private long finishTimeEpoch; - private DateFormat formatter; - - @Override - public void setConf(Configuration conf) { - super.setConf(conf); - try { - fs = FileSystem.get(conf); - } catch (IOException e) { - throw new RuntimeException(e); - } - conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); - conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); - formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - TEST_UTIL = new HBaseTestingUtility(conf); - ROOT_DIR = TEST_UTIL.getDataTestDir("TestHFilePerformance").toString(); - } - - public void startTime() { - startTimeEpoch = System.currentTimeMillis(); - System.out.println(formatTime() + " Started timing."); - } - - public void stopTime() { - finishTimeEpoch = System.currentTimeMillis(); - System.out.println(formatTime() + " Stopped timing."); - } - - public long getIntervalMillis() { - return finishTimeEpoch - startTimeEpoch; - } - - public void printlnWithTimestamp(String message) { - System.out.println(formatTime() + " " + message); - } - - /* - * Format millis into minutes and seconds. - */ - public String formatTime(long milis){ - return formatter.format(milis); - } - - public String formatTime(){ - return formatTime(System.currentTimeMillis()); - } - - private FSDataOutputStream createFSOutput(Path name) throws IOException { - if (fs.exists(name)) - fs.delete(name, true); - FSDataOutputStream fout = fs.create(name); - return fout; - } - - //TODO have multiple ways of generating key/value e.g. dictionary words - //TODO to have a sample compressable data, for now, made 1 out of 3 values random - // keys are all random. - - private static class KeyValueGenerator { - Random keyRandomizer; - Random valueRandomizer; - long randomValueRatio = 3; // 1 out of randomValueRatio generated values will be random. - long valueSequence = 0 ; - - - KeyValueGenerator() { - keyRandomizer = new Random(0L); //TODO with seed zero - valueRandomizer = new Random(1L); //TODO with seed one - } - - // Key is always random now. - void getKey(byte[] key) { - keyRandomizer.nextBytes(key); - } - - void getValue(byte[] value) { - if (valueSequence % randomValueRatio == 0) - valueRandomizer.nextBytes(value); - valueSequence++; - } - } - - /** - * - * @param fileType "HFile" or "SequenceFile" - * @param keyLength - * @param valueLength - * @param codecName "none", "lzo", "gz", "snappy" - * @param cipherName "none", "aes" - * @param rows number of rows to be written. - * @param writeMethod used for HFile only. - * @param minBlockSize used for HFile only. - * @throws IOException - */ - //TODO writeMethod: implement multiple ways of writing e.g. A) known length (no chunk) B) using a buffer and streaming (for many chunks). - public void timeWrite(String fileType, int keyLength, int valueLength, - String codecName, String cipherName, long rows, String writeMethod, int minBlockSize) - throws IOException { - System.out.println("File Type: " + fileType); - System.out.println("Writing " + fileType + " with codecName: " + codecName + - " cipherName: " + cipherName); - long totalBytesWritten = 0; - - - //Using separate randomizer for key/value with seeds matching Sequence File. - byte[] key = new byte[keyLength]; - byte[] value = new byte[valueLength]; - KeyValueGenerator generator = new KeyValueGenerator(); - - startTime(); - - Path path = new Path(ROOT_DIR, fileType + ".Performance"); - System.out.println(ROOT_DIR + Path.SEPARATOR + path.getName()); - FSDataOutputStream fout = createFSOutput(path); - - if ("HFile".equals(fileType)){ - HFileContextBuilder builder = new HFileContextBuilder() - .withCompression(AbstractHFileWriter.compressionByName(codecName)) - .withBlockSize(minBlockSize); - if (cipherName != "none") { - byte[] cipherKey = new byte[AES.KEY_LENGTH]; - new SecureRandom().nextBytes(cipherKey); - builder.withEncryptionContext( - Encryption.newContext(conf) - .setCipher(Encryption.getCipher(conf, cipherName)) - .setKey(cipherKey)); - } - HFileContext context = builder.build(); - System.out.println("HFile write method: "); - HFile.Writer writer = HFile.getWriterFactoryNoCache(conf) - .withOutputStream(fout) - .withFileContext(context) - .withComparator(new KeyValue.RawBytesComparator()) - .create(); - - // Writing value in one shot. - for (long l=0; l