From cb8e69a80cecb95abdfc93a787bea0bedef275ed Mon Sep 17 00:00:00 2001 From: cnauroth Date: Thu, 14 May 2015 22:22:24 -0700 Subject: [PATCH] HADOOP-11960. Enable Azure-Storage Client Side logging. Contributed by Dushyanth. --- .../hadoop-common/CHANGES.txt | 3 + .../fs/azure/AzureNativeFileSystemStore.java | 5 + ...estNativeAzureFileSystemClientLogging.java | 130 ++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 2f8acb0c79a..aecfde4cfbe 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -569,6 +569,9 @@ Release 2.8.0 - UNRELEASED HADOOP-11713. ViewFileSystem should support snapshot methods. (Rakesh R via cnauroth) + HADOOP-11960. Enable Azure-Storage Client Side logging. + (Dushyanth via cnauroth) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 5dc096397d2..3267d8bc422 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -148,6 +148,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { private static final String KEY_SELF_THROTTLE_READ_FACTOR = "fs.azure.selfthrottling.read.factor"; private static final String KEY_SELF_THROTTLE_WRITE_FACTOR = "fs.azure.selfthrottling.write.factor"; + private static final String KEY_ENABLE_STORAGE_CLIENT_LOGGING = "fs.azure.storage.client.logging"; + private static final String PERMISSION_METADATA_KEY = "hdi_permission"; private static final String OLD_PERMISSION_METADATA_KEY = "asv_permission"; private static final String IS_FOLDER_METADATA_KEY = "hdi_isfolder"; @@ -681,6 +683,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { selfThrottlingWriteFactor = sessionConfiguration.getFloat( KEY_SELF_THROTTLE_WRITE_FACTOR, DEFAULT_SELF_THROTTLE_WRITE_FACTOR); + OperationContext.setLoggingEnabledByDefault(sessionConfiguration. + getBoolean(KEY_ENABLE_STORAGE_CLIENT_LOGGING, false)); + if (LOG.isDebugEnabled()) { LOG.debug(String .format( diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java new file mode 100644 index 00000000000..da39fa3dbbb --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.net.URI; +import java.util.StringTokenizer; + +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils.LogCapturer; +import org.apache.log4j.Logger; +import org.junit.Test; + +/** + * Test to validate Azure storage client side logging. Tests works only when + * testing with Live Azure storage because Emulator does not have support for + * client-side logging. + * + */ +public class TestNativeAzureFileSystemClientLogging extends + NativeAzureFileSystemBaseTest { + + private AzureBlobStorageTestAccount testAccount; + + // Core-site config controlling Azure Storage Client logging + private static final String KEY_LOGGING_CONF_STRING = "fs.azure.storage.client.logging"; + + // Temporary directory created using WASB. + private static final String TEMP_DIR = "tempDir"; + + /* + * Helper method to verify the client logging is working. This check primarily + * checks to make sure we see a line in the logs corresponding to the entity + * that is created during test run. + */ + private boolean verifyStorageClientLogs(String capturedLogs, String entity) + throws Exception { + + URI uri = testAccount.getRealAccount().getBlobEndpoint(); + String container = testAccount.getRealContainer().getName(); + String validateString = uri + Path.SEPARATOR + container + Path.SEPARATOR + + entity; + boolean entityFound = false; + + StringTokenizer tokenizer = new StringTokenizer(capturedLogs, "\n"); + + while (tokenizer.hasMoreTokens()) { + String token = tokenizer.nextToken(); + if (token.contains(validateString)) { + entityFound = true; + break; + } + } + return entityFound; + } + + /* + * Helper method that updates the core-site config to enable/disable logging. + */ + private void updateFileSystemConfiguration(Boolean loggingFlag) + throws Exception { + + Configuration conf = fs.getConf(); + conf.set(KEY_LOGGING_CONF_STRING, loggingFlag.toString()); + URI uri = fs.getUri(); + fs.initialize(uri, conf); + } + + // Using WASB code to communicate with Azure Storage. + private void performWASBOperations() throws Exception { + + Path tempDir = new Path(Path.SEPARATOR + TEMP_DIR); + fs.mkdirs(tempDir); + fs.delete(tempDir, true); + } + + @Test + public void testLoggingEnabled() throws Exception { + + LogCapturer logs = LogCapturer.captureLogs(new Log4JLogger(Logger + .getRootLogger())); + + // Update configuration based on the Test. + updateFileSystemConfiguration(true); + + performWASBOperations(); + + assertTrue(verifyStorageClientLogs(logs.getOutput(), TEMP_DIR)); + } + + @Test + public void testLoggingDisabled() throws Exception { + + LogCapturer logs = LogCapturer.captureLogs(new Log4JLogger(Logger + .getRootLogger())); + + // Update configuration based on the Test. + updateFileSystemConfiguration(false); + + performWASBOperations(); + + assertFalse(verifyStorageClientLogs(logs.getOutput(), TEMP_DIR)); + } + + @Override + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + testAccount = AzureBlobStorageTestAccount.create(); + return testAccount; + } +} \ No newline at end of file