From d9a9e9913ef4c7a36775c1b5bdb14c77339918ce Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Thu, 20 Jun 2019 09:42:45 -0700 Subject: [PATCH] HDDS-1579. Create OMDoubleBuffer metrics. (#871) --- .../om/ratis/OzoneManagerDoubleBuffer.java | 43 +++++++-- .../OzoneManagerDoubleBufferMetrics.java | 89 +++++++++++++++++++ .../ozone/om/ratis/metrics/package-info.java | 21 +++++ ...eManagerDoubleBufferWithDummyResponse.java | 31 +++++-- 4 files changed, 173 insertions(+), 11 deletions(-) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/OzoneManagerDoubleBufferMetrics.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/package-info.java diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java index 8c25347449c..68be333a672 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java @@ -23,15 +23,17 @@ import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.ratis.helpers.DoubleBufferEntry; +import org.apache.hadoop.ozone.om.ratis.metrics.OzoneManagerDoubleBufferMetrics; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.utils.db.BatchOperation; - import org.apache.ratis.util.ExitUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * This class implements DoubleBuffer implementation of OMClientResponse's. In @@ -63,6 +65,8 @@ public class OzoneManagerDoubleBuffer { private final AtomicLong flushedTransactionCount = new AtomicLong(0); private final AtomicLong flushIterations = new AtomicLong(0); private volatile boolean isRunning; + private OzoneManagerDoubleBufferMetrics ozoneManagerDoubleBufferMetrics; + private long maxFlushedTransactionsInOneIteration; private final OzoneManagerRatisSnapshot ozoneManagerRatisSnapShot; @@ -71,8 +75,9 @@ public class OzoneManagerDoubleBuffer { this.currentBuffer = new ConcurrentLinkedQueue<>(); this.readyBuffer = new ConcurrentLinkedQueue<>(); this.omMetadataManager = omMetadataManager; - this.ozoneManagerRatisSnapShot = ozoneManagerRatisSnapShot; + this.ozoneManagerDoubleBufferMetrics = + OzoneManagerDoubleBufferMetrics.create(); isRunning = true; // Daemon thread which runs in back ground and flushes transactions to DB. @@ -80,7 +85,6 @@ public class OzoneManagerDoubleBuffer { daemon.setName("OMDoubleBufferFlushThread"); daemon.start(); - } /** @@ -120,6 +124,7 @@ public class OzoneManagerDoubleBuffer { .max(Long::compareTo).get(); readyBuffer.clear(); + // cleanup cache. cleanupCache(lastRatisTransactionIndex); @@ -129,6 +134,9 @@ public class OzoneManagerDoubleBuffer { // update the last updated index in OzoneManagerStateMachine. ozoneManagerRatisSnapShot.updateLastAppliedIndex( lastRatisTransactionIndex); + + // set metrics. + updateMetrics(flushedTransactionsSize); } } catch (InterruptedException ex) { Thread.currentThread().interrupt(); @@ -162,6 +170,23 @@ public class OzoneManagerDoubleBuffer { omMetadataManager.getUserTable().cleanupCache(lastRatisTransactionIndex); } + /** + * Update OzoneManagerDoubleBuffer metrics values. + * @param flushedTransactionsSize + */ + private void updateMetrics( + long flushedTransactionsSize) { + ozoneManagerDoubleBufferMetrics.incrTotalNumOfFlushOperations(); + ozoneManagerDoubleBufferMetrics.incrTotalSizeOfFlushedTransactions( + flushedTransactionsSize); + if (maxFlushedTransactionsInOneIteration < flushedTransactionsSize) { + maxFlushedTransactionsInOneIteration = flushedTransactionsSize; + ozoneManagerDoubleBufferMetrics + .setMaxNumberOfTransactionsFlushedInOneIteration( + flushedTransactionsSize); + } + } + /** * Stop OM DoubleBuffer flush thread. */ @@ -170,6 +195,9 @@ public class OzoneManagerDoubleBuffer { LOG.info("Stopping OMDoubleBuffer flush thread"); isRunning = false; daemon.interrupt(); + + // stop metrics. + ozoneManagerDoubleBufferMetrics.unRegister(); } else { LOG.info("OMDoubleBuffer flush thread is not running."); } @@ -236,5 +264,10 @@ public class OzoneManagerDoubleBuffer { readyBuffer = temp; } + @VisibleForTesting + public OzoneManagerDoubleBufferMetrics getOzoneManagerDoubleBufferMetrics() { + return ozoneManagerDoubleBufferMetrics; + } + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/OzoneManagerDoubleBufferMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/OzoneManagerDoubleBufferMetrics.java new file mode 100644 index 00000000000..e2d7f72e44c --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/OzoneManagerDoubleBufferMetrics.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.ratis.metrics; + +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; + +/** + * Class which maintains metrics related to OzoneManager DoubleBuffer. + */ +public class OzoneManagerDoubleBufferMetrics { + + private static final String SOURCE_NAME = + OzoneManagerDoubleBufferMetrics.class.getSimpleName(); + + @Metric(about = "Total Number of flush operations happened in " + + "OzoneManagerDoubleBuffer.") + private MutableCounterLong totalNumOfFlushOperations; + + @Metric(about = "Total Number of flushed transactions happened in " + + "OzoneManagerDoubleBuffer.") + private MutableCounterLong totalNumOfFlushedTransactions; + + @Metric(about = "Max Number of transactions flushed in a iteration in " + + "OzoneManagerDoubleBuffer. This will provide a value which is maximum " + + "number of transactions flushed in a single flush iteration till now.") + private MutableCounterLong maxNumberOfTransactionsFlushedInOneIteration; + + + public static OzoneManagerDoubleBufferMetrics create() { + MetricsSystem ms = DefaultMetricsSystem.instance(); + return ms.register(SOURCE_NAME, + "OzoneManager DoubleBuffer Metrics", + new OzoneManagerDoubleBufferMetrics()); + } + + public void incrTotalNumOfFlushOperations() { + this.totalNumOfFlushOperations.incr(); + } + + public void incrTotalSizeOfFlushedTransactions( + long flushedTransactions) { + this.totalNumOfFlushedTransactions.incr(flushedTransactions); + } + + public void setMaxNumberOfTransactionsFlushedInOneIteration( + long maxTransactions) { + // We should set the value with maxTransactions, so decrement old value + // first and then add the new value. + this.maxNumberOfTransactionsFlushedInOneIteration.incr( + Math.negateExact(getMaxNumberOfTransactionsFlushedInOneIteration()) + + maxTransactions); + } + + public long getTotalNumOfFlushOperations() { + return totalNumOfFlushOperations.value(); + } + + public long getTotalNumOfFlushedTransactions() { + return totalNumOfFlushedTransactions.value(); + } + + public long getMaxNumberOfTransactionsFlushedInOneIteration() { + return maxNumberOfTransactionsFlushedInOneIteration.value(); + } + + public void unRegister() { + MetricsSystem ms = DefaultMetricsSystem.instance(); + ms.unregisterSource(SOURCE_NAME); + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/package-info.java new file mode 100644 index 00000000000..e41c645b581 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/metrics/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * package which contains metrics classes. + */ +package org.apache.hadoop.ozone.om.ratis.metrics; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java index 116595500c0..e5499222f5b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerDoubleBufferWithDummyResponse.java @@ -22,11 +22,6 @@ import java.io.IOException; import java.util.UUID; import java.util.concurrent.atomic.AtomicLong; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos - .CreateBucketResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos - .OMResponse; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -34,7 +29,12 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .CreateBucketResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos + .OMResponse; +import org.apache.hadoop.ozone.om.ratis.metrics.OzoneManagerDoubleBufferMetrics; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; @@ -91,6 +91,18 @@ public class TestOzoneManagerDoubleBufferWithDummyResponse { public void testDoubleBufferWithDummyResponse() throws Exception { String volumeName = UUID.randomUUID().toString(); int bucketCount = 100; + OzoneManagerDoubleBufferMetrics ozoneManagerDoubleBufferMetrics = + doubleBuffer.getOzoneManagerDoubleBufferMetrics(); + + // As we have not flushed/added any transactions, all metrics should have + // value zero. + Assert.assertTrue(ozoneManagerDoubleBufferMetrics + .getTotalNumOfFlushOperations() == 0); + Assert.assertTrue(ozoneManagerDoubleBufferMetrics + .getTotalNumOfFlushedTransactions() == 0); + Assert.assertTrue(ozoneManagerDoubleBufferMetrics + .getMaxNumberOfTransactionsFlushedInOneIteration() == 0); + for (int i=0; i < bucketCount; i++) { doubleBuffer.add(createDummyBucketResponse(volumeName, UUID.randomUUID().toString()), trxId.incrementAndGet()); @@ -98,6 +110,13 @@ public class TestOzoneManagerDoubleBufferWithDummyResponse { GenericTestUtils.waitFor(() -> doubleBuffer.getFlushedTransactionCount() == bucketCount, 100, 60000); + + Assert.assertTrue(ozoneManagerDoubleBufferMetrics + .getTotalNumOfFlushOperations() > 0); + Assert.assertTrue(ozoneManagerDoubleBufferMetrics + .getTotalNumOfFlushedTransactions() == bucketCount); + Assert.assertTrue(ozoneManagerDoubleBufferMetrics + .getMaxNumberOfTransactionsFlushedInOneIteration() > 0); Assert.assertTrue(omMetadataManager.countRowsInTable( omMetadataManager.getBucketTable()) == (bucketCount)); Assert.assertTrue(doubleBuffer.getFlushIterations() > 0);