diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 9870fc71029..07b3ad0e864 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -71,6 +71,9 @@ Release 2.7.0 - UNRELEASED HADOOP-10840. Fix OutOfMemoryError caused by metrics system in Azure File System. (Shanyu Zhao via cnauroth) + HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics + collection. (Shanyu Zhao via cnauroth) + OPTIMIZATIONS HADOOP-11323. WritableComparator#compare keeps reference to byte array. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 076c48aa662..ad2e2e6635c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -641,6 +641,8 @@ public class NativeAzureFileSystem extends FileSystem { static final String AZURE_OUTPUT_STREAM_BUFFER_SIZE_PROPERTY_NAME = "fs.azure.output.stream.buffer.size"; + public static final String SKIP_AZURE_METRICS_PROPERTY_NAME = "fs.azure.skip.metrics"; + private class NativeAzureFsInputStream extends FSInputStream { private InputStream in; private final String key; @@ -1035,13 +1037,15 @@ public class NativeAzureFileSystem extends FileSystem { store = createDefaultStore(conf); } - // Make sure the metrics system is available before interacting with Azure - AzureFileSystemMetricsSystem.fileSystemStarted(); - metricsSourceName = newMetricsSourceName(); - String sourceDesc = "Azure Storage Volume File System metrics"; instrumentation = new AzureFileSystemInstrumentation(conf); - AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc, + if(!conf.getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) { + // Make sure the metrics system is available before interacting with Azure + AzureFileSystemMetricsSystem.fileSystemStarted(); + metricsSourceName = newMetricsSourceName(); + String sourceDesc = "Azure Storage Volume File System metrics"; + AzureFileSystemMetricsSystem.registerSource(metricsSourceName, sourceDesc, instrumentation); + } store.initialize(uri, conf, instrumentation); setConf(conf); @@ -2207,8 +2211,10 @@ public class NativeAzureFileSystem extends FileSystem { long startTime = System.currentTimeMillis(); - AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName); - AzureFileSystemMetricsSystem.fileSystemClosed(); + if(!getConf().getBoolean(SKIP_AZURE_METRICS_PROPERTY_NAME, false)) { + AzureFileSystemMetricsSystem.unregisterSource(metricsSourceName); + AzureFileSystemMetricsSystem.fileSystemClosed(); + } if (LOG.isDebugEnabled()) { LOG.debug("Submitting metrics when file system closed took " diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java index f44613d91d4..7820b7e65d5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java @@ -74,4 +74,15 @@ public class TestNativeAzureFileSystemMetricsSystem { assertTrue(name2.startsWith("AzureFileSystemMetrics")); assertTrue(!name1.equals(name2)); } + + @Test + public void testSkipMetricsCollection() throws Exception { + AzureBlobStorageTestAccount a; + a = AzureBlobStorageTestAccount.createMock(); + a.getFileSystem().getConf().setBoolean( + NativeAzureFileSystem.SKIP_AZURE_METRICS_PROPERTY_NAME, true); + a.getFileSystem().create(new Path("/foo")).close(); + a.closeFileSystem(); // Causes the file system to close, which publishes metrics + assertEquals(0, getFilesCreated(a)); + } }