From 3bef7c80a97709b367781180b2e11fc50653d3c8 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Fri, 15 May 2015 11:05:01 +0530 Subject: [PATCH] HDFS-6888. Allow selectively audit logging ops (Contributed by Chen He) --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 1 + .../hdfs/server/namenode/FSNamesystem.java | 11 +- .../src/main/resources/hdfs-default.xml | 9 ++ .../server/namenode/TestAuditLogAtDebug.java | 131 ++++++++++++++++++ 5 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogAtDebug.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 445b7c205d6..6c0923c0520 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -552,6 +552,8 @@ Release 2.8.0 - UNRELEASED HDFS-8350. Remove old webhdfs.xml and other outdated documentation stuff. (Brahma Reddy Battula via aajisaka) + HDFS-6888. Allow selectively audit logging ops (Chen He via vinayakumarb) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index ae056fab20f..1d0cf4b36ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -341,6 +341,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_DEFAULT = false; public static final String DFS_NAMENODE_AUDIT_LOG_ASYNC_KEY = "dfs.namenode.audit.log.async"; public static final boolean DFS_NAMENODE_AUDIT_LOG_ASYNC_DEFAULT = false; + public static final String DFS_NAMENODE_AUDIT_LOG_DEBUG_CMDLIST = "dfs.namenode.audit.log.debug.cmdlist"; public static final String DFS_BALANCER_MOVEDWINWIDTH_KEY = "dfs.balancer.movedWinWidth"; public static final long DFS_BALANCER_MOVEDWINWIDTH_DEFAULT = 5400*1000L; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 33aaa72f0c1..4d82fabf7ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -8149,15 +8149,20 @@ void checkAccess(String src, FsAction mode) throws IOException { * defined in the config file. It can also be explicitly listed in the * config file. */ - private static class DefaultAuditLogger extends HdfsAuditLogger { + @VisibleForTesting + static class DefaultAuditLogger extends HdfsAuditLogger { private boolean logTokenTrackingId; + private Set debugCmdSet = new HashSet(); @Override public void initialize(Configuration conf) { logTokenTrackingId = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_KEY, DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_DEFAULT); + + debugCmdSet.addAll(Arrays.asList(conf.getTrimmedStrings( + DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_DEBUG_CMDLIST))); } @Override @@ -8165,7 +8170,9 @@ public void logAuditEvent(boolean succeeded, String userName, InetAddress addr, String cmd, String src, String dst, FileStatus status, UserGroupInformation ugi, DelegationTokenSecretManager dtSecretManager) { - if (auditLog.isInfoEnabled()) { + + if (auditLog.isDebugEnabled() || + (auditLog.isInfoEnabled() && !debugCmdSet.contains(cmd))) { final StringBuilder sb = auditBuffer.get(); sb.setLength(0); sb.append("allowed=").append(succeeded).append("\t"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 5d1d670564d..7f0730b07f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2084,6 +2084,15 @@ + + dfs.namenode.audit.log.debug.cmdlist + + + A comma separated list of NameNode commands that are written to the HDFS + namenode audit log only if the audit log level is debug. + + + dfs.client.use.legacy.blockreader.local false diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogAtDebug.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogAtDebug.java new file mode 100644 index 00000000000..ce115147ae6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogAtDebug.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.namenode; + +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.DefaultAuditLogger; +import org.apache.log4j.Level; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.net.Inet4Address; +import java.util.Arrays; +import java.util.List; + +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.*; + +/** + * Test that the HDFS Audit logger respects DFS_NAMENODE_AUDIT_LOG_DEBUG_CMDLIST. + */ +public class TestAuditLogAtDebug { + static final Log LOG = LogFactory.getLog(TestAuditLogAtDebug.class); + + @Rule + public Timeout timeout = new Timeout(300000); + + private static final String DUMMY_COMMAND_1 = "dummycommand1"; + private static final String DUMMY_COMMAND_2 = "dummycommand2"; + + private DefaultAuditLogger makeSpyLogger( + Level level, Optional> debugCommands) { + DefaultAuditLogger logger = new DefaultAuditLogger(); + Configuration conf = new HdfsConfiguration(); + if (debugCommands.isPresent()) { + conf.set(DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_DEBUG_CMDLIST, + Joiner.on(",").join(debugCommands.get())); + } + logger.initialize(conf); + ((Log4JLogger) FSNamesystem.auditLog).getLogger().setLevel(level); + return spy(logger); + } + + private void logDummyCommandToAuditLog(HdfsAuditLogger logger, String command) { + logger.logAuditEvent(true, "", + Inet4Address.getLoopbackAddress(), + command, "", "", + null, null, null); + } + + @Test + public void testDebugCommandNotLoggedAtInfo() { + DefaultAuditLogger logger = + makeSpyLogger( + Level.INFO, Optional.of(Arrays.asList(DUMMY_COMMAND_1))); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_1); + verify(logger, never()).logAuditMessage(anyString()); + } + + @Test + public void testDebugCommandLoggedAtDebug() { + DefaultAuditLogger logger = + makeSpyLogger( + Level.DEBUG, Optional.of(Arrays.asList(DUMMY_COMMAND_1))); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_1); + verify(logger, times(1)).logAuditMessage(anyString()); + } + + @Test + public void testInfoCommandLoggedAtInfo() { + DefaultAuditLogger logger = + makeSpyLogger( + Level.INFO, Optional.of(Arrays.asList(DUMMY_COMMAND_1))); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_2); + verify(logger, times(1)).logAuditMessage(anyString()); + } + + @Test + public void testMultipleDebugCommandsNotLoggedAtInfo() { + DefaultAuditLogger logger = + makeSpyLogger( + Level.INFO, + Optional.of(Arrays.asList(DUMMY_COMMAND_1, DUMMY_COMMAND_2))); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_1); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_2); + verify(logger, never()).logAuditMessage(anyString()); + } + + @Test + public void testMultipleDebugCommandsLoggedAtDebug() { + DefaultAuditLogger logger = + makeSpyLogger( + Level.DEBUG, + Optional.of(Arrays.asList(DUMMY_COMMAND_1, DUMMY_COMMAND_2))); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_1); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_2); + verify(logger, times(2)).logAuditMessage(anyString()); + } + + @Test + public void testEmptyDebugCommands() { + DefaultAuditLogger logger = makeSpyLogger( + Level.INFO, Optional.>absent()); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_1); + logDummyCommandToAuditLog(logger, DUMMY_COMMAND_2); + verify(logger, times(2)).logAuditMessage(anyString()); + } +}