From d303b7f40fc2e6bd285bea788d6341ad59da5d25 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Tue, 26 Sep 2017 19:22:18 +0800 Subject: [PATCH] HDFS-12524. Ozone: Record number of keys scanned and hinted for getRangeKVs call. Contributed by Weiwei Yang. --- .../org/apache/hadoop/utils/LevelDBStore.java | 20 +++++++++--- .../hadoop/utils/MetadataKeyFilters.java | 32 +++++++++++++++++-- .../org/apache/hadoop/utils/RocksDBStore.java | 13 +++++++- .../hadoop/ozone/TestMetadataStore.java | 2 ++ 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java index 8b0d9050829..4aa26ca8b93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java @@ -353,11 +353,23 @@ private List> getRangeKVs(byte[] startKey, if (dbIter != null) { dbIter.close(); } - long end = System.currentTimeMillis(); - long timeConsumed = end - start; if (LOG.isDebugEnabled()) { - LOG.debug("Time consumed for getRangeKVs() is {}," - + " result length is {}.", timeConsumed, result.size()); + if (filters != null) { + for (MetadataKeyFilters.MetadataKeyFilter filter : filters) { + int scanned = filter.getKeysScannedNum(); + int hinted = filter.getKeysHintedNum(); + if (scanned > 0 || hinted > 0) { + LOG.debug( + "getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}", + filter.getClass().getSimpleName(), filter.getKeysScannedNum(), + filter.getKeysHintedNum()); + } + } + } + long end = System.currentTimeMillis(); + long timeConsumed = end - start; + LOG.debug("Time consumed for getRangeKVs() is {}ms," + + " result length is {}.", timeConsumed, result.size()); } } return result; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java index 1688ece4b95..f98d077604a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java @@ -38,6 +38,14 @@ public interface MetadataKeyFilter { * @return true if a certain condition satisfied, return false otherwise. */ boolean filterKey(byte[] preKey, byte[] currentKey, byte[] nextKey); + + default int getKeysScannedNum() { + return 0; + } + + default int getKeysHintedNum() { + return 0; + } } /** @@ -47,19 +55,37 @@ public interface MetadataKeyFilter { public static class KeyPrefixFilter implements MetadataKeyFilter { private String keyPrefix = null; + private int keysScanned = 0; + private int keysHinted = 0; public KeyPrefixFilter(String keyPrefix) { this.keyPrefix = keyPrefix; } - @Override public boolean filterKey(byte[] preKey, byte[] currentKey, + @Override + public boolean filterKey(byte[] preKey, byte[] currentKey, byte[] nextKey) { + keysScanned++; if (Strings.isNullOrEmpty(keyPrefix)) { return true; } else { - return currentKey != null && - DFSUtil.bytes2String(currentKey).startsWith(keyPrefix); + if (currentKey != null && + DFSUtil.bytes2String(currentKey).startsWith(keyPrefix)) { + keysHinted++; + return true; + } + return false; } } + + @Override + public int getKeysScannedNum() { + return keysScanned; + } + + @Override + public int getKeysHintedNum() { + return keysHinted; + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java index 56a79a1fa46..54da94cd5e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java @@ -201,11 +201,22 @@ private List> getRangeKVs(byte[] startKey, long end = System.currentTimeMillis(); long timeConsumed = end - start; if (LOG.isDebugEnabled()) { + if (filters != null) { + for (MetadataKeyFilters.MetadataKeyFilter filter : filters) { + int scanned = filter.getKeysScannedNum(); + int hinted = filter.getKeysHintedNum(); + if (scanned > 0 || hinted > 0) { + LOG.debug( + "getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}", + filter.getClass().getSimpleName(), filter.getKeysScannedNum(), + filter.getKeysHintedNum()); + } + } + } LOG.debug("Time consumed for getRangeKVs() is {}ms," + " result length is {}.", timeConsumed, result.size()); } } - return result; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java index f4757bbcd8d..6a918606418 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestMetadataStore.java @@ -265,6 +265,8 @@ public void testGetRangeKVs() throws IOException { Assert.assertTrue(result.stream().allMatch(entry -> new String(entry.getKey()).startsWith("b") )); + Assert.assertEquals(20, filter1.getKeysScannedNum()); + Assert.assertEquals(10, filter1.getKeysHintedNum()); result = store.getRangeKVs(null, 3, filter1); Assert.assertEquals(3, result.size()); result = store.getRangeKVs(getBytes("b3"), 1, filter1);