HBASE-15135 Add metrics for storefile age

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -155,6 +155,14 @@ public interface MetricsRegionServerSource extends BaseSource {
   String MEMSTORE_SIZE = "memStoreSize";
   String MEMSTORE_SIZE_DESC = "Size of the memstore";
   String STOREFILE_SIZE = "storeFileSize";
+  String MAX_STORE_FILE_AGE = "maxStoreFileAge";
+  String MIN_STORE_FILE_AGE = "minStoreFileAge";
+  String AVG_STORE_FILE_AGE = "avgStoreFileAge";
+  String NUM_REFERENCE_FILES = "numReferenceFiles";
+  String MAX_STORE_FILE_AGE_DESC = "Max age of store files hosted on this region server";
+  String MIN_STORE_FILE_AGE_DESC = "Min age of store files hosted on this region server";
+  String AVG_STORE_FILE_AGE_DESC = "Average age of store files hosted on this region server";
+  String NUM_REFERENCE_FILES_DESC = "Number of reference file on this region server";
   String STOREFILE_SIZE_DESC = "Size of storefiles being served.";
   String TOTAL_REQUEST_COUNT = "totalRequestCount";
   String TOTAL_REQUEST_COUNT_DESC =

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java

@@ -97,6 +97,26 @@ public interface MetricsRegionServerWrapper {
    */
   long getStoreFileSize();
 
+  /**
+   * @return Max age of store files hosted on this region server
+   */
+  long getMaxStoreFileAge();
+
+  /**
+   * @return Min age of store files hosted on this region server
+   */
+  long getMinStoreFileAge();
+
+  /**
+   *  @return Average age of store files hosted on this region server
+   */
+  long getAvgStoreFileAge();
+
+  /**
+   *  @return Number of reference files on this region server
+   */
+  long getNumReferenceFiles();
+
   /**
    * Get the number of requests per second.
    */

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java

@@ -74,6 +74,26 @@ public interface MetricsRegionWrapper {
    */
   long getFilteredReadRequestCount();
 
+  /**
+   * @return Max age of store files under this region
+   */
+  long getMaxStoreFileAge();
+
+  /**
+   * @return Min age of store files under this region
+   */
+  long getMinStoreFileAge();
+
+  /**
+   *  @return Average age of store files under this region
+   */
+  long getAvgStoreFileAge();
+
+  /**
+   *  @return Number of reference files under this region
+   */
+  long getNumReferenceFiles();
+
   /**
    * Get the total number of mutations that have been issued against this region.
    */

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java

@@ -195,6 +195,14 @@ public class MetricsRegionServerSourceImpl
           .addGauge(Interns.info(STOREFILE_COUNT, STOREFILE_COUNT_DESC), rsWrap.getNumStoreFiles())
           .addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemstoreSize())
           .addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize())
+          .addGauge(Interns.info(MAX_STORE_FILE_AGE, MAX_STORE_FILE_AGE_DESC),
+              rsWrap.getMaxStoreFileAge())
+          .addGauge(Interns.info(MIN_STORE_FILE_AGE, MIN_STORE_FILE_AGE_DESC),
+              rsWrap.getMinStoreFileAge())
+          .addGauge(Interns.info(AVG_STORE_FILE_AGE, AVG_STORE_FILE_AGE_DESC),
+              rsWrap.getAvgStoreFileAge())
+          .addGauge(Interns.info(NUM_REFERENCE_FILES, NUM_REFERENCE_FILES_DESC),
+              rsWrap.getNumReferenceFiles())
           .addGauge(Interns.info(RS_START_TIME_NAME, RS_START_TIME_DESC),
               rsWrap.getStartCode())
           .addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC),

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java

@@ -216,6 +216,22 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
               regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE,
               MetricsRegionServerSource.MEMSTORE_SIZE_DESC),
           this.regionWrapper.getMemstoreSize());
+      mrb.addGauge(Interns.info(
+        regionNamePrefix + MetricsRegionServerSource.MAX_STORE_FILE_AGE,
+        MetricsRegionServerSource.MAX_STORE_FILE_AGE_DESC),
+        this.regionWrapper.getMaxStoreFileAge());
+      mrb.addGauge(Interns.info(
+        regionNamePrefix + MetricsRegionServerSource.MIN_STORE_FILE_AGE,
+        MetricsRegionServerSource.MIN_STORE_FILE_AGE_DESC),
+        this.regionWrapper.getMinStoreFileAge());
+      mrb.addGauge(Interns.info(
+        regionNamePrefix + MetricsRegionServerSource.AVG_STORE_FILE_AGE,
+        MetricsRegionServerSource.AVG_STORE_FILE_AGE_DESC),
+        this.regionWrapper.getAvgStoreFileAge());
+      mrb.addGauge(Interns.info(
+        regionNamePrefix + MetricsRegionServerSource.NUM_REFERENCE_FILES,
+        MetricsRegionServerSource.NUM_REFERENCE_FILES_DESC),
+        this.regionWrapper.getNumReferenceFiles());
       mrb.addGauge(Interns.info(
               regionNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE,
               MetricsRegionServerSource.STOREFILE_SIZE_DESC),

hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java

@@ -111,6 +111,26 @@ public class TestMetricsRegionSourceImpl {
       return 0;
     }
 
+    @Override
+    public long getMaxStoreFileAge() {
+      return 0;
+    }
+
+    @Override
+    public long getMinStoreFileAge() {
+      return 0;
+    }
+
+    @Override
+    public long getAvgStoreFileAge() {
+      return 0;
+    }
+
+    @Override
+    public long getNumReferenceFiles() {
+      return 0;
+    }
+
     @Override
     public long getWriteRequestCount() {
       return 0;

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java

@@ -1887,6 +1887,89 @@ public class HStore implements Store {
     return this.storeEngine.getStoreFileManager().getStorefileCount();
   }
 
+  @Override
+  public long getMaxStoreFileAge() {
+    long earliestTS = Long.MAX_VALUE;
+    for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
+      StoreFile.Reader r = s.getReader();
+      if (r == null) {
+        LOG.warn("StoreFile " + s + " has a null Reader");
+        continue;
+      }
+      if (!s.isHFile()) {
+        continue;
+      }
+      long createdTS = s.getFileInfo().getCreatedTimestamp();
+      earliestTS = (createdTS < earliestTS) ? createdTS : earliestTS;
+    }
+    long now = EnvironmentEdgeManager.currentTime();
+    return now - earliestTS;
+  }
+
+  @Override
+  public long getMinStoreFileAge() {
+    long latestTS = 0;
+    for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
+      StoreFile.Reader r = s.getReader();
+      if (r == null) {
+        LOG.warn("StoreFile " + s + " has a null Reader");
+        continue;
+      }
+      if (!s.isHFile()) {
+        continue;
+      }
+      long createdTS = s.getFileInfo().getCreatedTimestamp();
+      latestTS = (createdTS > latestTS) ? createdTS : latestTS;
+    }
+    long now = EnvironmentEdgeManager.currentTime();
+    return now - latestTS;
+  }
+
+  @Override
+  public long getAvgStoreFileAge() {
+    long sum = 0, count = 0;
+    for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
+      StoreFile.Reader r = s.getReader();
+      if (r == null) {
+        LOG.warn("StoreFile " + s + " has a null Reader");
+        continue;
+      }
+      if (!s.isHFile()) {
+        continue;
+      }
+      sum += s.getFileInfo().getCreatedTimestamp();
+      count++;
+    }
+    if (count == 0) {
+      return 0;
+    }
+    long avgTS = sum / count;
+    long now = EnvironmentEdgeManager.currentTime();
+    return now - avgTS;
+  }
+
+  @Override
+  public long getNumReferenceFiles() {
+    long numRefFiles = 0;
+    for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
+      if (s.isReference()) {
+        numRefFiles++;
+      }
+    }
+    return numRefFiles;
+  }
+
+  @Override
+  public long getNumHFiles() {
+    long numHFiles = 0;
+    for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
+      if (s.isHFile()) {
+        numHFiles++;
+      }
+    }
+    return numHFiles;
+  }
+
   @Override
   public long getStoreSizeUncompressed() {
     return this.totalUncompressedBytes;

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

@@ -66,6 +66,10 @@ class MetricsRegionServerWrapperImpl
   private volatile long numStoreFiles = 0;
   private volatile long memstoreSize = 0;
   private volatile long storeFileSize = 0;
+  private volatile long maxStoreFileAge = 0;
+  private volatile long minStoreFileAge = 0;
+  private volatile long avgStoreFileAge = 0;
+  private volatile long numReferenceFiles = 0;
   private volatile double requestsPerSecond = 0.0;
   private volatile long readRequestsCount = 0;
   private volatile long filteredReadRequestsCount = 0;
@@ -389,6 +393,26 @@ class MetricsRegionServerWrapperImpl
     return numStoreFiles;
   }
 
+  @Override
+  public long getMaxStoreFileAge() {
+    return maxStoreFileAge;
+  }
+
+  @Override
+  public long getMinStoreFileAge() {
+    return minStoreFileAge;
+  }
+
+  @Override
+  public long getAvgStoreFileAge() {
+    return avgStoreFileAge;
+  }
+
+  @Override
+  public long getNumReferenceFiles() {
+    return numReferenceFiles;
+  }
+
   @Override
   public long getMemstoreSize() {
     return memstoreSize;
@@ -614,6 +638,9 @@ class MetricsRegionServerWrapperImpl
             new HDFSBlocksDistribution();
 
         long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize = 0;
+        long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0;
+        long avgAgeNumerator = 0, numHFiles = 0;
+        long tempMinStoreFileAge = Long.MAX_VALUE;
         long tempReadRequestsCount = 0, tempFilteredReadRequestsCount = 0,
           tempWriteRequestsCount = 0;
         long tempCheckAndMutateChecksFailed = 0;
@@ -657,6 +684,20 @@ class MetricsRegionServerWrapperImpl
             tempNumStoreFiles += store.getStorefilesCount();
             tempMemstoreSize += store.getMemStoreSize();
             tempStoreFileSize += store.getStorefilesSize();
+
+            long storeMaxStoreFileAge = store.getMaxStoreFileAge();
+            tempMaxStoreFileAge = (storeMaxStoreFileAge > tempMaxStoreFileAge) ?
+              storeMaxStoreFileAge : tempMaxStoreFileAge;
+
+            long storeMinStoreFileAge = store.getMinStoreFileAge();
+            tempMinStoreFileAge = (storeMinStoreFileAge < tempMinStoreFileAge) ?
+              storeMinStoreFileAge : tempMinStoreFileAge;
+
+            long storeHFiles = store.getNumHFiles();
+            avgAgeNumerator += store.getAvgStoreFileAge() * storeHFiles;
+            numHFiles += storeHFiles;
+            tempNumReferenceFiles += store.getNumReferenceFiles();
+
             tempStorefileIndexSize += store.getStorefilesIndexSize();
             tempTotalStaticBloomSize += store.getTotalStaticBloomSize();
             tempTotalStaticIndexSize += store.getTotalStaticIndexSize();
@@ -723,6 +764,16 @@ class MetricsRegionServerWrapperImpl
         numStoreFiles = tempNumStoreFiles;
         memstoreSize = tempMemstoreSize;
         storeFileSize = tempStoreFileSize;
+        maxStoreFileAge = tempMaxStoreFileAge;
+        if (tempMinStoreFileAge != Long.MAX_VALUE) {
+          minStoreFileAge = tempMinStoreFileAge;
+        }
+
+        if (numHFiles != 0) {
+          avgStoreFileAge = avgAgeNumerator / numHFiles;
+        }
+
+        numReferenceFiles= tempNumReferenceFiles;
         readRequestsCount = tempReadRequestsCount;
         filteredReadRequestsCount = tempFilteredReadRequestsCount;
         writeRequestsCount = tempWriteRequestsCount;

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java

@@ -43,6 +43,10 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
   private long numStoreFiles;
   private long memstoreSize;
   private long storeFileSize;
+  private long maxStoreFileAge;
+  private long minStoreFileAge;
+  private long avgStoreFileAge;
+  private long numReferenceFiles;
 
   private ScheduledFuture<?> regionMetricsUpdateTask;
 
@@ -136,6 +140,26 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
     return this.region.compactionsFinished.get();
   }
 
+  @Override
+  public long getMaxStoreFileAge() {
+    return maxStoreFileAge;
+  }
+
+  @Override
+  public long getMinStoreFileAge() {
+    return minStoreFileAge;
+  }
+
+  @Override
+  public long getAvgStoreFileAge() {
+    return avgStoreFileAge;
+  }
+
+  @Override
+  public long getNumReferenceFiles() {
+    return numReferenceFiles;
+  }
+
   @Override
   public int getRegionHashCode() {
     return this.region.hashCode();
@@ -148,18 +172,46 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
       long tempNumStoreFiles = 0;
       long tempMemstoreSize = 0;
       long tempStoreFileSize = 0;
+      long tempMaxStoreFileAge = 0;
+      long tempMinStoreFileAge = Long.MAX_VALUE;
+      long tempNumReferenceFiles = 0;
 
+      long avgAgeNumerator = 0;
+      long numHFiles = 0;
       if (region.stores != null) {
         for (Store store : region.stores.values()) {
           tempNumStoreFiles += store.getStorefilesCount();
           tempMemstoreSize += store.getMemStoreSize();
           tempStoreFileSize += store.getStorefilesSize();
+
+          long storeMaxStoreFileAge = store.getMaxStoreFileAge();
+          tempMaxStoreFileAge = (storeMaxStoreFileAge > tempMaxStoreFileAge) ?
+            storeMaxStoreFileAge : tempMaxStoreFileAge;
+
+          long storeMinStoreFileAge = store.getMinStoreFileAge();
+          tempMinStoreFileAge = (storeMinStoreFileAge < tempMinStoreFileAge) ?
+            storeMinStoreFileAge : tempMinStoreFileAge;
+
+          long storeHFiles = store.getNumHFiles();
+          avgAgeNumerator += store.getAvgStoreFileAge() * storeHFiles;
+          numHFiles += storeHFiles;
+          tempNumReferenceFiles += store.getNumReferenceFiles();
         }
       }
 
       numStoreFiles = tempNumStoreFiles;
       memstoreSize = tempMemstoreSize;
       storeFileSize = tempStoreFileSize;
+      maxStoreFileAge = tempMaxStoreFileAge;
+      if (tempMinStoreFileAge != Long.MAX_VALUE) {
+        minStoreFileAge = tempMinStoreFileAge;
+      }
+
+      if (numHFiles != 0) {
+        avgStoreFileAge = avgAgeNumerator / numHFiles;
+      }
+
+      numReferenceFiles = tempNumReferenceFiles;
     }
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java

@@ -345,6 +345,31 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
    */
   int getStorefilesCount();
 
+  /**
+   * @return Max age of store files in this store
+   */
+  long getMaxStoreFileAge();
+
+  /**
+   * @return Min age of store files in this store
+   */
+  long getMinStoreFileAge();
+
+  /**
+   *  @return Average age of store files in this store, 0 if no store files
+   */
+  long getAvgStoreFileAge();
+
+  /**
+   *  @return Number of reference files in this store
+   */
+  long getNumReferenceFiles();
+
+  /**
+   *  @return Number of HFiles in this store
+   */
+  long getNumHFiles();
+
   /**
    * @return The size of the store files, in bytes, uncompressed.
    */

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java

@@ -290,6 +290,13 @@ public class StoreFile {
     return this.fileInfo.isReference();
   }
 
+  /**
+   * @return True if this is HFile.
+   */
+  public boolean isHFile() {
+    return this.fileInfo.isHFile(this.fileInfo.getPath());
+  }
+
   /**
    * @return True if this file was made by a major compaction.
    */

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java

@@ -97,6 +97,9 @@ public class StoreFileInfo {
 
   private RegionCoprocessorHost coprocessorHost;
 
+  // timestamp on when the file was created, is 0 and ignored for reference or link files
+  private long createdTimestamp;
+
   /**
    * Create a Store File Info
    * @param conf the {@link Configuration} to use
@@ -132,6 +135,7 @@ public class StoreFileInfo {
               " reference to " + referencePath);
     } else if (isHFile(p)) {
       // HFile
+      this.createdTimestamp = fs.getFileStatus(initialPath).getModificationTime();
       this.reference = null;
       this.link = null;
     } else {
@@ -182,6 +186,7 @@ public class StoreFileInfo {
     this.fs = fs;
     this.conf = conf;
     this.initialPath = fileStatus.getPath();
+    this.createdTimestamp = fileStatus.getModificationTime();
     this.reference = reference;
     this.link = null;
   }
@@ -419,6 +424,13 @@ public class StoreFileInfo {
     return m.matches() && m.groupCount() > 1;
   }
 
+  /**
+   * @return timestamp when this file was created (as returned by filesystem)
+   */
+  public long getCreatedTimestamp() {
+    return createdTimestamp;
+  }
+
   /*
    * Return path to the file referred to by a Reference.  Presumes a directory
    * hierarchy of <code>${hbase.rootdir}/data/${namespace}/tablename/regionname/familyname</code>.

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java

@@ -70,6 +70,26 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
     return 1900;
   }
 
+  @Override
+  public long getMaxStoreFileAge() {
+    return 2;
+  }
+
+  @Override
+  public long getMinStoreFileAge() {
+    return 2;
+  }
+
+  @Override
+  public long getAvgStoreFileAge() {
+    return 2;
+  }
+
+  @Override
+  public long getNumReferenceFiles() {
+    return 2;
+  }
+
   @Override
   public double getRequestsPerSecond() {
     return 0;

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java

@@ -80,6 +80,26 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper {
     return 107;
   }
 
+  @Override
+  public long getMaxStoreFileAge() {
+    return 2;
+  }
+
+  @Override
+  public long getMinStoreFileAge() {
+    return 2;
+  }
+
+  @Override
+  public long getAvgStoreFileAge() {
+    return 2;
+  }
+
+  @Override
+  public long getNumReferenceFiles() {
+    return 2;
+  }
+
   @Override
   public long getWriteRequestCount() {
     return 106;

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java

@@ -42,6 +42,18 @@ public class TestMetricsRegion {
     HELPER.assertGauge(
       "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_storeFileCount",
       102, agg);
+    HELPER.assertGauge(
+      "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_maxStoreFileAge",
+      2, agg);
+    HELPER.assertGauge(
+      "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_minStoreFileAge",
+      2, agg);
+    HELPER.assertGauge(
+      "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_avgStoreFileAge",
+      2, agg);
+    HELPER.assertGauge(
+      "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_numReferenceFiles",
+      2, agg);
     HELPER.assertGauge(
       "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_memstoreSize",
       103, agg);

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java

@@ -60,6 +60,10 @@ public class TestMetricsRegionServer {
     HELPER.assertGauge("regionServerStartTime", 100, serverSource);
     HELPER.assertGauge("regionCount", 101, serverSource);
     HELPER.assertGauge("storeCount", 2, serverSource);
+    HELPER.assertGauge("maxStoreFileAge", 2, serverSource);
+    HELPER.assertGauge("minStoreFileAge", 2, serverSource);
+    HELPER.assertGauge("avgStoreFileAge", 2, serverSource);
+    HELPER.assertGauge("numReferenceFiles", 2, serverSource);
     HELPER.assertGauge("hlogFileCount", 10, serverSource);
     HELPER.assertGauge("hlogFileSize", 1024000, serverSource);
     HELPER.assertGauge("storeFileCount", 300, serverSource);

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java

@@ -238,6 +238,29 @@ public class TestRegionServerMetrics {
     t.close();
   }
 
+  @Test
+  public void testStoreFileAge() throws Exception {
+    TableName tableName = TableName.valueOf("testStoreFileAge");
+    byte[] cf = Bytes.toBytes("d");
+    byte[] row = Bytes.toBytes("rk");
+    byte[] qualifier = Bytes.toBytes("qual");
+    byte[] val = Bytes.toBytes("Value");
+
+    //Force a hfile.
+    Table t = TEST_UTIL.createTable(tableName, cf);
+    Put p = new Put(row);
+    p.addColumn(cf, qualifier, val);
+    t.put(p);
+    TEST_UTIL.getHBaseAdmin().flush(tableName);
+
+    metricsRegionServer.getRegionServerWrapper().forceRecompute();
+    assertTrue(metricsHelper.getGaugeLong("maxStoreFileAge", serverSource) > 0);
+    assertTrue(metricsHelper.getGaugeLong("minStoreFileAge", serverSource) > 0);
+    assertTrue(metricsHelper.getGaugeLong("avgStoreFileAge", serverSource) > 0);
+
+    t.close();
+  }
+
   @Test
   public void testCheckAndPutCount() throws Exception {
     String tableNameString = "testCheckAndPutCount";