From c0ef7e7680d882e2182f48f033109678a48742ab Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Tue, 3 Jul 2018 11:38:14 -0700 Subject: [PATCH] HDDS-198. Create AuditLogger mechanism to be used by OM, SCM and Datanode. Contributed by Dinesh Chitlangia. --- hadoop-hdds/common/pom.xml | 18 +++ .../hadoop/ozone/audit/AuditAction.java | 30 ++++ .../hadoop/ozone/audit/AuditEventStatus.java | 36 +++++ .../hadoop/ozone/audit/AuditLogger.java | 128 +++++++++++++++++ .../hadoop/ozone/audit/AuditLoggerType.java | 37 +++++ .../hadoop/ozone/audit/AuditMarker.java | 38 +++++ .../apache/hadoop/ozone/audit/Auditable.java | 32 +++++ .../hadoop/ozone/audit/package-info.java | 123 ++++++++++++++++ .../hadoop/ozone/audit/DummyAction.java | 51 +++++++ .../hadoop/ozone/audit/DummyEntity.java | 57 ++++++++ .../ozone/audit/TestOzoneAuditLogger.java | 131 ++++++++++++++++++ .../hadoop/ozone/audit/package-info.java | 23 +++ .../src/test/resources/log4j2.properties | 76 ++++++++++ .../apache/hadoop/ozone/audit/OMAction.java | 51 +++++++ .../hadoop/ozone/audit/package-info.java | 22 +++ 15 files changed, 853 insertions(+) create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditAction.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditEventStatus.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLoggerType.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMarker.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/Auditable.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyEntity.java create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/package-info.java create mode 100644 hadoop-hdds/common/src/test/resources/log4j2.properties create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml index a8a634c0a2c..406852280b5 100644 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@ -31,6 +31,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hdds true + 2.11.0 + 3.4.2 @@ -81,6 +83,22 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> test-jar + + org.apache.logging.log4j + log4j-api + ${log4j2.version} + + + org.apache.logging.log4j + log4j-core + ${log4j2.version} + + + com.lmax + disruptor + ${disruptor.version} + + diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditAction.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditAction.java new file mode 100644 index 00000000000..8c1d6f0c67d --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditAction.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.audit; + +/** + * Interface to define AuditAction. + */ +public interface AuditAction { + /** + * Implementation must override. + * @return String + */ + String getAction(); +} + diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditEventStatus.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditEventStatus.java new file mode 100644 index 00000000000..098ab6b2f7f --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditEventStatus.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.audit; + +/** + * Enum to define AuditEventStatus values. + */ +public enum AuditEventStatus { + SUCCESS("SUCCESS"), + FAILURE("FAILURE"); + + private String status; + + AuditEventStatus(String status){ + this.status = status; + } + + public String getStatus() { + return status; + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java new file mode 100644 index 00000000000..46ffaab9ef5 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.audit; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.StructuredDataMessage; +import org.apache.logging.log4j.spi.ExtendedLogger; + +import java.util.Map; + +/** + * Class to define Audit Logger for Ozone. + */ +public class AuditLogger { + + private ExtendedLogger logger; + + private static final String SUCCESS = AuditEventStatus.SUCCESS.getStatus(); + private static final String FAILURE = AuditEventStatus.FAILURE.getStatus(); + private static final String FQCN = AuditLogger.class.getName(); + private static final Marker WRITE_MARKER = AuditMarker.WRITE.getMarker(); + private static final Marker READ_MARKER = AuditMarker.READ.getMarker(); + + /** + * Parametrized Constructor to initialize logger. + * @param type + */ + public AuditLogger(AuditLoggerType type){ + initializeLogger(type); + } + + /** + * Initializes the logger with specific type. + * @param loggerType specified one of the values from enum AuditLoggerType. + */ + private void initializeLogger(AuditLoggerType loggerType){ + this.logger = LogManager.getContext(false).getLogger(loggerType.getType()); + } + + @VisibleForTesting + public ExtendedLogger getLogger() { + return logger; + } + + public void logWriteSuccess(AuditAction type, Map data) { + logWriteSuccess(type, data, Level.INFO); + } + + public void logWriteSuccess(AuditAction type, Map data, Level + level) { + StructuredDataMessage msg = new StructuredDataMessage("", SUCCESS, + type.getAction(), data); + this.logger.logIfEnabled(FQCN, level, WRITE_MARKER, msg, null); + } + + + public void logWriteFailure(AuditAction type, Map data) { + logWriteFailure(type, data, Level.INFO, null); + } + + public void logWriteFailure(AuditAction type, Map data, Level + level) { + logWriteFailure(type, data, level, null); + } + + public void logWriteFailure(AuditAction type, Map data, + Throwable exception) { + logWriteFailure(type, data, Level.INFO, exception); + } + + public void logWriteFailure(AuditAction type, Map data, Level + level, Throwable exception) { + StructuredDataMessage msg = new StructuredDataMessage("", FAILURE, + type.getAction(), data); + this.logger.logIfEnabled(FQCN, level, WRITE_MARKER, msg, exception); + } + + public void logReadSuccess(AuditAction type, Map data) { + logReadSuccess(type, data, Level.INFO); + } + + public void logReadSuccess(AuditAction type, Map data, Level + level) { + StructuredDataMessage msg = new StructuredDataMessage("", SUCCESS, + type.getAction(), data); + this.logger.logIfEnabled(FQCN, level, READ_MARKER, msg, null); + } + + public void logReadFailure(AuditAction type, Map data) { + logReadFailure(type, data, Level.INFO, null); + } + + public void logReadFailure(AuditAction type, Map data, Level + level) { + logReadFailure(type, data, level, null); + } + + public void logReadFailure(AuditAction type, Map data, + Throwable exception) { + logReadFailure(type, data, Level.INFO, exception); + } + + public void logReadFailure(AuditAction type, Map data, Level + level, Throwable exception) { + StructuredDataMessage msg = new StructuredDataMessage("", FAILURE, + type.getAction(), data); + this.logger.logIfEnabled(FQCN, level, READ_MARKER, msg, exception); + } + +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLoggerType.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLoggerType.java new file mode 100644 index 00000000000..18241c7712a --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLoggerType.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.audit; + +/** + * Enumeration for defining types of Audit Loggers in Ozone. + */ +public enum AuditLoggerType { + DNLOGGER("DNAudit"), + OMLOGGER("OMAudit"), + SCMLOGGER("SCMAudit"); + + private String type; + + public String getType() { + return type; + } + + AuditLoggerType(String type){ + this.type = type; + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMarker.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMarker.java new file mode 100644 index 00000000000..505b9580715 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMarker.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.audit; + +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; + +/** + * Defines audit marker types. + */ +public enum AuditMarker { + WRITE(MarkerManager.getMarker("WRITE")), + READ(MarkerManager.getMarker("READ")); + + private Marker marker; + + AuditMarker(Marker marker){ + this.marker = marker; + } + + public Marker getMarker(){ + return marker; + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/Auditable.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/Auditable.java new file mode 100644 index 00000000000..d388bca72f1 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/Auditable.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.audit; + +import java.util.Map; + +/** + * Interface to make an entity auditable. + */ +public interface Auditable { + /** + * Must override in implementation. + * @return Map with values to be logged in audit. + */ + Map toAuditMap(); +} + diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java new file mode 100644 index 00000000000..3743fddd4f2 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.audit; +/** + ****************************************************************************** + * Important + * 1. Any changes to classes in this package can render the logging + * framework broken. + * 2. The logger framework has been designed keeping in mind future + * plans to build a log parser. + * 3. Please exercise great caution when attempting changes in this package. + ****************************************************************************** + * + * + * This package lays the foundation for Audit logging in Ozone. + * AuditLogging in Ozone has been built using log4j2 which brings in new + * features that facilitate turning on/off selective audit events by using + * MarkerFilter, checking for change in logging configuration periodically + * and reloading the changes, use of disruptor framework for improved + * Asynchronous logging. + * + * The log4j2 configurations can be specified in XML, YAML, JSON and + * Properties file. For Ozone, we are using the Properties file due to sheer + * simplicity, readability and ease of modification. + * + * log4j2 configuration file can be passed to startup command with option + * -Dlog4j.configurationFile unlike -Dlog4j.configuration in log4j 1.x + * + ****************************************************************************** + * Understanding the Audit Logging framework in Ozone. + ****************************************************************************** + * **** Auditable *** + * This is an interface to mark an entity as auditable. + * This interface must be implemented by entities requiring audit logging. + * For example - KSMVolumeArgs, KSMBucketArgs. + * The implementing class must override toAuditMap() to return an + * instance of Map where both Key and Value are String. + * + * Key: must not contain any spaces. If the key is multi word then use + * camel case. + * Value: if it is a collection/array, then it must be converted to a comma + * delimited string + * + * *** AuditAction *** + * This is an interface to define the various type of actions to be audited. + * To ensure separation of concern, for each sub-component you must create an + * Enum to implement AuditAction. + * Structure of Enum can be referred from the test class DummyAction. + * + * For starters, we expect following 3 implementations of AuditAction: + * OMAction - to define action types for Ozone Manager + * SCMAction - to define action types for Storage Container manager + * DNAction - to define action types for Datanode + * + * *** AuditEventStatus *** + * Enum to define Audit event status like success and failure. + * This is used in AuditLogger.logXXX() methods. + * + * * *** AuditLogger *** + * This is where the audit logging magic unfolds. + * The class has 2 Markers defined - READ and WRITE. + * These markers are used to tag when logging events. + * + * *** AuditLoggerType *** + * Enum to define the various AuditLoggers in Ozone + * + * *** AuditMarker *** + * Enum to define various Audit Markers used in AuditLogging. + * + * **************************************************************************** + * Usage + * **************************************************************************** + * Using the AuditLogger to log events: + * 1. Get a logger by specifying the appropriate logger type + * Example: ExtendedLogger AUDIT = new AuditLogger(AuditLoggerType.OMLogger) + * + * 2. Log Read/Write and Success/Failure event as needed. + * Example + * AUDIT.logWriteSuccess(AuditAction type, Map data, Level + * level) + * + * If logging is done without specifying Level, then Level implicitly + * defaults to INFO + * AUDIT.logWriteSuccess(AuditAction type, Map data) + * + * See sample invocations in src/test in the following class: + * org.apache.hadoop.ozone.audit.TestOzoneAuditLogger + * + * **************************************************************************** + * Defining new Logger types + * **************************************************************************** + * New Logger type can be added with following steps: + * 1. Update AuditLoggerType to add the new type + * 2. Create new Enum by implementing AuditAction if needed + * 3. Ensure the required entity implements Auditable + * + * **************************************************************************** + * Defining new Marker types + * **************************************************************************** + * New Markers can be configured as follows: + * 1. Define new markers in AuditMarker + * 2. Get the Marker in AuditLogger for use in the log methods, example: + * private static final Marker WRITE_MARKER = AuditMarker.WRITE.getMarker(); + * 3. Define log methods in AuditLogger to use the new Marker type + * 4. Call these new methods from the required classes to audit with these + * new markers + * 5. The marker based filtering can be configured in log4j2 configurations + * Refer log4j2.properties in src/test/resources for a sample. + */ diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java new file mode 100644 index 00000000000..6044c0a2f55 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyAction.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.audit; + +/** + * Enum to define Dummy AuditAction Type for test. + */ +public enum DummyAction implements AuditAction { + + CREATE_VOLUME("CREATE_VOLUME"), + CREATE_BUCKET("CREATE_BUCKET"), + CREATE_KEY("CREATE_KEY"), + READ_VOLUME("READ_VOLUME"), + READ_BUCKET("READ_BUCKET"), + READ_KEY("READ_BUCKET"), + UPDATE_VOLUME("UPDATE_VOLUME"), + UPDATE_BUCKET("UPDATE_BUCKET"), + UPDATE_KEY("UPDATE_KEY"), + DELETE_VOLUME("DELETE_VOLUME"), + DELETE_BUCKET("DELETE_BUCKET"), + DELETE_KEY("DELETE_KEY"), + SET_OWNER("SET_OWNER"), + SET_QUOTA("SET_QUOTA"); + + private String action; + + DummyAction(String action) { + this.action = action; + } + + @Override + public String getAction() { + return this.action; + } + +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyEntity.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyEntity.java new file mode 100644 index 00000000000..0c2d98fab29 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/DummyEntity.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.audit; + +import java.util.HashMap; +import java.util.Map; + +/** + * DummyEntity that implements Auditable for test purpose. + */ +public class DummyEntity implements Auditable { + + private String key1; + private String key2; + + public DummyEntity(){ + this.key1 = "value1"; + this.key2 = "value2"; + } + public String getKey1() { + return key1; + } + + public void setKey1(String key1) { + this.key1 = key1; + } + + public String getKey2() { + return key2; + } + + public void setKey2(String key2) { + this.key2 = key2; + } + + @Override + public Map toAuditMap() { + Map auditMap = new HashMap<>(); + auditMap.put("key1", this.key1); + auditMap.put("key2", this.key2); + return auditMap; + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java new file mode 100644 index 00000000000..d3cc9e4ce8a --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.audit; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.Level; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.assertTrue; + +/** + * Test Ozone Audit Logger. + */ +public class TestOzoneAuditLogger { + + private static final Logger LOG = LoggerFactory.getLogger + (TestOzoneAuditLogger.class.getName()); + private static AuditLogger AUDIT = new AuditLogger(AuditLoggerType.OMLOGGER); + public DummyEntity auditableObj = new DummyEntity(); + + @BeforeClass + public static void setUp(){ + System.setProperty("log4j.configurationFile", "log4j2.properties"); + } + + @AfterClass + public static void tearDown() { + File file = new File("audit.log"); + if (FileUtils.deleteQuietly(file)) { + LOG.info(file.getName() + + " has been deleted as all tests have completed."); + } else { + LOG.info("audit.log could not be deleted."); + } + } + + /** + * Ensures WriteSuccess events are logged @ INFO and above. + */ + @Test + public void logInfoWriteSuccess() throws IOException { + AUDIT.logWriteSuccess(DummyAction.CREATE_VOLUME, auditableObj.toAuditMap(), Level.INFO); + String expected = "[INFO ] OMAudit - CREATE_VOLUME [ key1=\"value1\" " + + "key2=\"value2\"] SUCCESS"; + verifyLog(expected); + } + + /** + * Test to verify default log level is INFO + */ + @Test + public void verifyDefaultLogLevel() throws IOException { + AUDIT.logWriteSuccess(DummyAction.CREATE_VOLUME, auditableObj.toAuditMap()); + String expected = "[INFO ] OMAudit - CREATE_VOLUME [ key1=\"value1\" " + + "key2=\"value2\"] SUCCESS"; + verifyLog(expected); + } + + /** + * Test to verify WriteFailure events are logged as ERROR. + */ + @Test + public void logErrorWriteFailure() throws IOException { + AUDIT.logWriteFailure(DummyAction.CREATE_VOLUME, auditableObj.toAuditMap(), Level.ERROR); + String expected = "[ERROR] OMAudit - CREATE_VOLUME [ key1=\"value1\" " + + "key2=\"value2\"] FAILURE"; + verifyLog(expected); + } + + /** + * Test to verify no READ event is logged. + */ + @Test + public void notLogReadEvents() throws IOException { + AUDIT.logReadSuccess(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.INFO); + AUDIT.logReadFailure(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.INFO); + AUDIT.logReadFailure(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.ERROR); + AUDIT.logReadFailure(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.ERROR, + new Exception("test")); + verifyLog(null); + } + + /** + * Test to ensure DEBUG level messages are not logged when INFO is enabled. + */ + @Test + public void notLogDebugEvents() throws IOException { + AUDIT.logWriteSuccess(DummyAction.CREATE_VOLUME, auditableObj.toAuditMap(), Level.DEBUG); + AUDIT.logReadSuccess(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.DEBUG); + verifyLog(null); + } + + public void verifyLog(String expected) throws IOException { + File file = new File("audit.log"); + List lines = FileUtils.readLines(file, (String)null); + if(expected == null){ + // When no log entry is expected, the log file must be empty + assertTrue(lines.size() == 0); + } else { + // When log entry is expected, the log file will contain one line and + // that must be equal to the expected string + assertTrue(expected.equalsIgnoreCase(lines.get(0))); + //empty the file + lines.remove(0); + FileUtils.writeLines(file, lines, false); + } + } +} diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/package-info.java new file mode 100644 index 00000000000..1222ad04e08 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/package-info.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.audit; +/** + * Unit tests of Ozone Audit Logger. + * For test purpose, the log4j2 configuration is loaded from file at: + * src/test/resources/log4j2.properties + */ diff --git a/hadoop-hdds/common/src/test/resources/log4j2.properties b/hadoop-hdds/common/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..d60df187eb1 --- /dev/null +++ b/hadoop-hdds/common/src/test/resources/log4j2.properties @@ -0,0 +1,76 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +#

+# http://www.apache.org/licenses/LICENSE-2.0 +#

+# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS,WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +name=PropertiesConfig + +# Checks for config change periodically and reloads +monitorInterval=5 + +filter=read, write +# filter.read.onMatch = DENY avoids logging all READ events +# filter.read.onMatch = ACCEPT permits logging all READ events +# The above two settings ignore the log levels in configuration +# filter.read.onMatch = NEUTRAL permits logging of only those READ events +# which are attempted at log level equal or greater than log level specified +# in the configuration +filter.read.type = MarkerFilter +filter.read.marker = READ +filter.read.onMatch = DENY +filter.read.onMismatch = NEUTRAL + +# filter.write.onMatch = DENY avoids logging all WRITE events +# filter.write.onMatch = ACCEPT permits logging all WRITE events +# The above two settings ignore the log levels in configuration +# filter.write.onMatch = NEUTRAL permits logging of only those WRITE events +# which are attempted at log level equal or greater than log level specified +# in the configuration +filter.write.type = MarkerFilter +filter.write.marker = WRITE +filter.write.onMatch = NEUTRAL +filter.write.onMismatch = NEUTRAL + +# Log Levels are organized from most specific to least: +# OFF (most specific, no logging) +# FATAL (most specific, little data) +# ERROR +# WARN +# INFO +# DEBUG +# TRACE (least specific, a lot of data) +# ALL (least specific, all data) + +appenders = console, audit +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%-5level] %c{1} - %msg%n + +appender.audit.type = File +appender.audit.name = AUDITLOG +appender.audit.fileName=audit.log +appender.audit.layout.type=PatternLayout +appender.audit.layout.pattern= [%-5level] %c{1} - %msg%n + +loggers=audit +logger.audit.type=AsyncLogger +logger.audit.name=OMAudit +logger.audit.level = INFO +logger.audit.appenderRefs = audit +logger.audit.appenderRef.file.ref = AUDITLOG + +rootLogger.level = INFO +rootLogger.appenderRefs = stdout +rootLogger.appenderRef.stdout.ref = STDOUT diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java new file mode 100644 index 00000000000..d780ea2c93b --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.audit; + +/** + * Enum to define OM Action types for Audit. + */ +public enum OMAction implements AuditAction { + + CREATE_VOLUME("CREATE_VOLUME"), + CREATE_BUCKET("CREATE_BUCKET"), + CREATE_KEY("CREATE_KEY"), + READ_VOLUME("READ_VOLUME"), + READ_BUCKET("READ_BUCKET"), + READ_KEY("READ_BUCKET"), + UPDATE_VOLUME("UPDATE_VOLUME"), + UPDATE_BUCKET("UPDATE_BUCKET"), + UPDATE_KEY("UPDATE_KEY"), + DELETE_VOLUME("DELETE_VOLUME"), + DELETE_BUCKET("DELETE_BUCKET"), + DELETE_KEY("DELETE_KEY"), + SET_OWNER("SET_OWNER"), + SET_QUOTA("SET_QUOTA"); + + private String action; + + OMAction(String action) { + this.action = action; + } + + @Override + public String getAction() { + return this.action; + } + +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java new file mode 100644 index 00000000000..0f887909d49 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.audit; +/** + * This package defines OMAction - an implementation of AuditAction + * OMAction defines audit action types for various actions that will be + * audited in OzoneManager. + */