containerIt = controller.getContainers();
- long count = 0;
-
- while (!stopping && containerIt.hasNext()) {
- Container container = containerIt.next();
- try {
- scrub(container);
- } catch (IOException e) {
- LOG.info("Unexpected error while scrubbing container {}",
- container.getContainerData().getContainerID());
- }
- count++;
- }
-
- LOG.debug("iterator ran integrity checks on {} containers", count);
- }
-
@VisibleForTesting
public void scrub(Container container) throws IOException {
if (!container.scanMetaData()) {
+ metrics.incNumUnHealthyContainers();
controller.markContainerUnhealthy(
container.getContainerData().getContainerID());
}
}
+ @VisibleForTesting
+ public ContainerMetadataScrubberMetrics getMetrics() {
+ return metrics;
+ }
+
public synchronized void shutdown() {
this.stopping = true;
this.interrupt();
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScrubberMetrics.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScrubberMetrics.java
new file mode 100644
index 00000000000..11629671fb3
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerMetadataScrubberMetrics.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.ozoneimpl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableCounterInt;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+
+/**
+ * This class captures the container meta-data scrubber metrics on the
+ * data-node.
+ **/
+@InterfaceAudience.Private
+@Metrics(about="DataNode container data scrubber metrics", context="dfs")
+public final class ContainerMetadataScrubberMetrics {
+ private final String name;
+ private final MetricsSystem ms;
+ @Metric("number of containers scanned in the current iteration")
+ private MutableGaugeInt numContainersScanned;
+ @Metric("number of unhealthy containers found in the current iteration")
+ private MutableGaugeInt numUnHealthyContainers;
+ @Metric("number of iterations of scanner completed since the restart")
+ private MutableCounterInt numScanIterations;
+
+ public int getNumContainersScanned() {
+ return numContainersScanned.value();
+ }
+
+ public void incNumContainersScanned() {
+ numContainersScanned.incr();
+ }
+
+ public void resetNumContainersScanned() {
+ numContainersScanned.decr(getNumContainersScanned());
+ }
+
+ public int getNumUnHealthyContainers() {
+ return numUnHealthyContainers.value();
+ }
+
+ public void incNumUnHealthyContainers() {
+ numUnHealthyContainers.incr();
+ }
+
+ public void resetNumUnhealthyContainers() {
+ numUnHealthyContainers.decr(getNumUnHealthyContainers());
+ }
+
+ public int getNumScanIterations() {
+ return numScanIterations.value();
+ }
+
+ public void incNumScanIterations() {
+ numScanIterations.incr();
+ }
+
+ public void unregister() {
+ ms.unregisterSource(name);
+ }
+
+ private ContainerMetadataScrubberMetrics(String name, MetricsSystem ms) {
+ this.name = name;
+ this.ms = ms;
+ }
+
+ public static ContainerMetadataScrubberMetrics create(Configuration conf) {
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ String name = "ContainerDataScrubberMetrics";
+ return ms.register(name, null,
+ new ContainerMetadataScrubberMetrics(name, ms));
+ }
+
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 8b9dc5713e7..1106e4670cb 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -178,14 +178,14 @@ public class OzoneContainer {
LOG.info("Background container scanner has been disabled.");
} else {
if (this.metadataScanner == null) {
- this.metadataScanner = new ContainerMetadataScanner(controller,
+ this.metadataScanner = new ContainerMetadataScanner(config, controller,
metadataScanInterval);
}
this.metadataScanner.start();
dataScanners = new ArrayList<>();
for (HddsVolume v : volumeSet.getVolumesList()) {
- ContainerDataScanner s = new ContainerDataScanner(controller,
+ ContainerDataScanner s = new ContainerDataScanner(config, controller,
v, bytesPerSec);
s.start();
dataScanners.add(s);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScrubberMetrics.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScrubberMetrics.java
new file mode 100644
index 00000000000..fe2c705f90c
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerScrubberMetrics.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.ozoneimpl;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdfs.util.Canceler;
+import org.apache.hadoop.hdfs.util.DataTransferThrottler;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.Iterator;
+
+/**
+ * This test verifies the container scrubber metrics functionality.
+ */
+public class TestContainerScrubberMetrics {
+ @Test
+ public void testContainerMetaDataScrubberMetrics() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ ContainerScrubberConfiguration c = conf.getObject(
+ ContainerScrubberConfiguration.class);
+ c.setMetadataScanInterval(0);
+ HddsVolume vol = Mockito.mock(HddsVolume.class);
+ ContainerController cntrl = mockContainerController(vol);
+
+ ContainerMetadataScanner mc = new ContainerMetadataScanner(conf,
+ cntrl, c.getMetadataScanInterval());
+ mc.runIteration();
+
+ Assert.assertEquals(1, mc.getMetrics().getNumScanIterations());
+ Assert.assertEquals(3, mc.getMetrics().getNumContainersScanned());
+ Assert.assertEquals(1, mc.getMetrics().getNumUnHealthyContainers());
+ }
+
+ @Test
+ public void testContainerDataScrubberMetrics() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ ContainerScrubberConfiguration c = conf.getObject(
+ ContainerScrubberConfiguration.class);
+ HddsVolume vol = Mockito.mock(HddsVolume.class);
+ ContainerController cntrl = mockContainerController(vol);
+
+ ContainerDataScanner sc = new ContainerDataScanner(conf, cntrl,
+ vol, c.getBandwidthPerVolume());
+ sc.runIteration();
+
+ ContainerDataScrubberMetrics m = sc.getMetrics();
+ Assert.assertEquals(1, m.getNumScanIterations());
+ Assert.assertEquals(2, m.getNumContainersScanned());
+ Assert.assertEquals(1, m.getNumUnHealthyContainers());
+ }
+
+ private ContainerController mockContainerController(HddsVolume vol) {
+ // healthy container
+ Container c1 = Mockito.mock(Container.class);
+ Mockito.when(c1.shouldScanData()).thenReturn(true);
+ Mockito.when(c1.scanMetaData()).thenReturn(true);
+ Mockito.when(c1.scanData(
+ Mockito.any(DataTransferThrottler.class),
+ Mockito.any(Canceler.class))).thenReturn(true);
+
+ // unhealthy container (corrupt data)
+ ContainerData c2d = Mockito.mock(ContainerData.class);
+ Mockito.when(c2d.getContainerID()).thenReturn(101L);
+ Container c2 = Mockito.mock(Container.class);
+ Mockito.when(c2.scanMetaData()).thenReturn(true);
+ Mockito.when(c2.shouldScanData()).thenReturn(true);
+ Mockito.when(c2.scanData(
+ Mockito.any(DataTransferThrottler.class),
+ Mockito.any(Canceler.class))).thenReturn(false);
+ Mockito.when(c2.getContainerData()).thenReturn(c2d);
+
+ // unhealthy container (corrupt metadata)
+ ContainerData c3d = Mockito.mock(ContainerData.class);
+ Mockito.when(c3d.getContainerID()).thenReturn(102L);
+ Container c3 = Mockito.mock(Container.class);
+ Mockito.when(c3.shouldScanData()).thenReturn(false);
+ Mockito.when(c3.scanMetaData()).thenReturn(false);
+ Mockito.when(c3.getContainerData()).thenReturn(c3d);
+
+ Iterator iter = Mockito.mock(Iterator.class);
+ Mockito.when(iter.hasNext()).thenReturn(true, true, true, false);
+ Mockito.when(iter.next()).thenReturn(c1, c2, c3);
+
+ ContainerController cntrl = Mockito.mock(ContainerController.class);
+ Mockito.when(cntrl.getContainers(vol))
+ .thenReturn(iter);
+ Mockito.when(cntrl.getContainers())
+ .thenReturn(iter);
+
+ return cntrl;
+ }
+
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java
index e4d52621698..863a2b3359f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scrubber/TestDataScrubber.java
@@ -159,7 +159,7 @@ public class TestDataScrubber {
deleteDirectory(chunksDir);
Assert.assertFalse(chunksDir.exists());
- ContainerMetadataScanner sb = new ContainerMetadataScanner(
+ ContainerMetadataScanner sb = new ContainerMetadataScanner(ozoneConfig,
oc.getController(), 0);
sb.scrub(c);