HDDS-98. Adding Ozone Manager Audit Log. Contributed by Dinesh Chitlangia.

This commit is contained in:
Nanda kumar 2018-09-05 00:11:07 +05:30
parent b9932162e9
commit 6bbd249011
10 changed files with 468 additions and 12 deletions

View File

@ -31,3 +31,40 @@ LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm #BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
LOG4J2.PROPERTIES_monitorInterval=30
LOG4J2.PROPERTIES_filter=read,write
LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
LOG4J2.PROPERTIES_filter.read.marker=READ
LOG4J2.PROPERTIES_filter.read.onMatch=DENY
LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
LOG4J2.PROPERTIES_filter.write.marker=WRITE
LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
LOG4J2.PROPERTIES_appenders=console, rolling
LOG4J2.PROPERTIES_appender.console.type=Console
LOG4J2.PROPERTIES_appender.console.name=STDOUT
LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
LOG4J2.PROPERTIES_loggers=audit
LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
LOG4J2.PROPERTIES_logger.audit.name=OMAudit
LOG4J2.PROPERTIES_logger.audit.level=INFO
LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
LOG4J2.PROPERTIES_rootLogger.level=INFO
LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT

View File

@ -180,4 +180,36 @@ public final class OzoneConsts {
public static final String CHUNKS_PATH = "chunksPath"; public static final String CHUNKS_PATH = "chunksPath";
public static final String CONTAINER_DB_TYPE = "containerDBType"; public static final String CONTAINER_DB_TYPE = "containerDBType";
public static final String CHECKSUM = "checksum"; public static final String CHECKSUM = "checksum";
// For OM Audit usage
public static final String VOLUME = "volume";
public static final String BUCKET = "bucket";
public static final String KEY = "key";
public static final String QUOTA = "quota";
public static final String QUOTA_IN_BYTES = "quotaInBytes";
public static final String CLIENT_ID = "clientID";
public static final String OWNER = "owner";
public static final String ADMIN = "admin";
public static final String USERNAME = "username";
public static final String PREV_KEY = "prevKey";
public static final String START_KEY = "startKey";
public static final String MAX_KEYS = "maxKeys";
public static final String PREFIX = "prefix";
public static final String KEY_PREFIX = "keyPrefix";
public static final String ACLS = "acls";
public static final String USER_ACL = "userAcl";
public static final String ADD_ACLS = "addAcls";
public static final String REMOVE_ACLS = "removeAcls";
public static final String MAX_NUM_OF_BUCKETS = "maxNumOfBuckets";
public static final String TO_KEY_NAME = "toKeyName";
public static final String STORAGE_TYPE = "storageType";
public static final String IS_VERSION_ENABLED = "isVersionEnabled";
public static final String CREATION_TIME = "creationTime";
public static final String DATA_SIZE = "dataSize";
public static final String REPLICATION_TYPE = "replicationType";
public static final String REPLICATION_FACTOR = "replicationFactor";
public static final String KEY_LOCATION_INFO = "keyLocationInfo";
} }

View File

@ -97,6 +97,8 @@ function ozonecmd_case
om) om)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManager HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManager
HDFS_OM_OPTS="${HDFS_OM_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/om-audit-log4j2.properties"
HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_OM_OPTS}"
;; ;;
oz) oz)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.Shell HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.Shell

View File

@ -0,0 +1,86 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with this
# work for additional information regarding copyright ownership. The ASF
# licenses this file to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# <p>
# http://www.apache.org/licenses/LICENSE-2.0
# <p>
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
#
name=PropertiesConfig
# Checks for config change periodically and reloads
monitorInterval=30
filter=read,write
# filter.read.onMatch=DENY avoids logging all READ events
# filter.read.onMatch=ACCEPT permits logging all READ events
# The above two settings ignore the log levels in configuration
# filter.read.onMatch=NEUTRAL permits logging of only those READ events
# which are attempted at log level equal or greater than log level specified
# in the configuration
filter.read.type=MarkerFilter
filter.read.marker=READ
filter.read.onMatch=DENY
filter.read.onMismatch=NEUTRAL
# filter.write.onMatch=DENY avoids logging all WRITE events
# filter.write.onMatch=ACCEPT permits logging all WRITE events
# The above two settings ignore the log levels in configuration
# filter.write.onMatch=NEUTRAL permits logging of only those WRITE events
# which are attempted at log level equal or greater than log level specified
# in the configuration
filter.write.type=MarkerFilter
filter.write.marker=WRITE
filter.write.onMatch=NEUTRAL
filter.write.onMismatch=NEUTRAL
# Log Levels are organized from most specific to least:
# OFF (most specific, no logging)
# FATAL (most specific, little data)
# ERROR
# WARN
# INFO
# DEBUG
# TRACE (least specific, a lot of data)
# ALL (least specific, all data)
appenders=console, rolling
appender.console.type=Console
appender.console.name=STDOUT
appender.console.layout.type=PatternLayout
appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
#Rolling File Appender with size & time thresholds.
#Rolling is triggered when either threshold is breached.
#The rolled over file is compressed by default
#Time interval is specified in seconds 86400s=1 day
appender.rolling.type=RollingFile
appender.rolling.name=RollingFile
appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
appender.rolling.layout.type=PatternLayout
appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
appender.rolling.policies.type=Policies
appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
appender.rolling.policies.time.interval=86400
appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
appender.rolling.policies.size.size=64MB
loggers=audit
logger.audit.type=AsyncLogger
logger.audit.name=OMAudit
logger.audit.level=INFO
logger.audit.appenderRefs=rolling
logger.audit.appenderRef.file.ref=RollingFile
rootLogger.level=INFO
rootLogger.appenderRefs=stdout
rootLogger.appenderRef.stdout.ref=STDOUT

View File

@ -18,24 +18,34 @@
package org.apache.hadoop.ozone.audit; package org.apache.hadoop.ozone.audit;
/** /**
* Enum to define OM Action types for Audit. * Enum to define Audit Action types for OzoneManager.
*/ */
public enum OMAction implements AuditAction { public enum OMAction implements AuditAction {
// WRITE Actions
ALLOCATE_BLOCK("ALLOCATE_BLOCK"),
ALLOCATE_KEY("ALLOCATE_KEY"),
COMMIT_KEY("COMMIT_KEY"),
CREATE_VOLUME("CREATE_VOLUME"), CREATE_VOLUME("CREATE_VOLUME"),
CREATE_BUCKET("CREATE_BUCKET"), CREATE_BUCKET("CREATE_BUCKET"),
CREATE_KEY("CREATE_KEY"), CREATE_KEY("CREATE_KEY"),
READ_VOLUME("READ_VOLUME"),
READ_BUCKET("READ_BUCKET"),
READ_KEY("READ_BUCKET"),
UPDATE_VOLUME("UPDATE_VOLUME"),
UPDATE_BUCKET("UPDATE_BUCKET"),
UPDATE_KEY("UPDATE_KEY"),
DELETE_VOLUME("DELETE_VOLUME"), DELETE_VOLUME("DELETE_VOLUME"),
DELETE_BUCKET("DELETE_BUCKET"), DELETE_BUCKET("DELETE_BUCKET"),
DELETE_KEY("DELETE_KEY"), DELETE_KEY("DELETE_KEY"),
RENAME_KEY("RENAME_KEY"),
SET_OWNER("SET_OWNER"), SET_OWNER("SET_OWNER"),
SET_QUOTA("SET_QUOTA"); SET_QUOTA("SET_QUOTA"),
UPDATE_VOLUME("UPDATE_VOLUME"),
UPDATE_BUCKET("UPDATE_BUCKET"),
UPDATE_KEY("UPDATE_KEY"),
// READ Actions
CHECK_VOLUME_ACCESS("CHECK_VOLUME_ACCESS"),
LIST_BUCKETS("LIST_BUCKETS"),
LIST_VOLUMES("LIST_VOLUMES"),
LIST_KEYS("LIST_KEYS"),
READ_VOLUME("READ_VOLUME"),
READ_BUCKET("READ_BUCKET"),
READ_KEY("READ_BUCKET");
private String action; private String action;

View File

@ -17,13 +17,17 @@
*/ */
package org.apache.hadoop.ozone.om.helpers; package org.apache.hadoop.ozone.om.helpers;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.Auditable;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.BucketArgs; .OzoneManagerProtocolProtos.BucketArgs;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper; import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
@ -31,7 +35,7 @@ import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
/** /**
* A class that encapsulates Bucket Arguments. * A class that encapsulates Bucket Arguments.
*/ */
public final class OmBucketArgs { public final class OmBucketArgs implements Auditable {
/** /**
* Name of the volume in which the bucket belongs to. * Name of the volume in which the bucket belongs to.
*/ */
@ -135,6 +139,25 @@ public final class OmBucketArgs {
return new Builder(); return new Builder();
} }
@Override
public Map<String, String> toAuditMap() {
Map<String, String> auditMap = new LinkedHashMap<>();
auditMap.put(OzoneConsts.VOLUME, this.volumeName);
auditMap.put(OzoneConsts.BUCKET, this.bucketName);
if(this.addAcls != null){
auditMap.put(OzoneConsts.ADD_ACLS, this.addAcls.toString());
}
if(this.removeAcls != null){
auditMap.put(OzoneConsts.REMOVE_ACLS, this.removeAcls.toString());
}
auditMap.put(OzoneConsts.IS_VERSION_ENABLED,
String.valueOf(this.isVersionEnabled));
if(this.storageType != null){
auditMap.put(OzoneConsts.STORAGE_TYPE, this.storageType.name());
}
return auditMap;
}
/** /**
* Builder for OmBucketArgs. * Builder for OmBucketArgs.
*/ */

View File

@ -21,18 +21,22 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.Auditable;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.BucketInfo; .OzoneManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper; import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import java.util.LinkedHashMap;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
/** /**
* A class that encapsulates Bucket Info. * A class that encapsulates Bucket Info.
*/ */
public final class OmBucketInfo { public final class OmBucketInfo implements Auditable {
/** /**
* Name of the volume in which the bucket belongs to. * Name of the volume in which the bucket belongs to.
*/ */
@ -137,6 +141,21 @@ public final class OmBucketInfo {
return new Builder(); return new Builder();
} }
@Override
public Map<String, String> toAuditMap() {
Map<String, String> auditMap = new LinkedHashMap<>();
auditMap.put(OzoneConsts.VOLUME, this.volumeName);
auditMap.put(OzoneConsts.BUCKET, this.bucketName);
auditMap.put(OzoneConsts.ACLS,
(this.acls != null) ? this.acls.toString() : null);
auditMap.put(OzoneConsts.IS_VERSION_ENABLED,
String.valueOf(this.isVersionEnabled));
auditMap.put(OzoneConsts.STORAGE_TYPE,
(this.storageType != null) ? this.storageType.name() : null);
auditMap.put(OzoneConsts.CREATION_TIME, String.valueOf(this.creationTime));
return auditMap;
}
/** /**
* Builder for OmBucketInfo. * Builder for OmBucketInfo.
*/ */

View File

@ -18,14 +18,18 @@
package org.apache.hadoop.ozone.om.helpers; package org.apache.hadoop.ozone.om.helpers;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.Auditable;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* Args for key. Client use this to specify key's attributes on key creation * Args for key. Client use this to specify key's attributes on key creation
* (putKey()). * (putKey()).
*/ */
public final class OmKeyArgs { public final class OmKeyArgs implements Auditable {
private final String volumeName; private final String volumeName;
private final String bucketName; private final String bucketName;
private final String keyName; private final String keyName;
@ -82,6 +86,22 @@ public final class OmKeyArgs {
return locationInfoList; return locationInfoList;
} }
@Override
public Map<String, String> toAuditMap() {
Map<String, String> auditMap = new LinkedHashMap<>();
auditMap.put(OzoneConsts.VOLUME, this.volumeName);
auditMap.put(OzoneConsts.BUCKET, this.bucketName);
auditMap.put(OzoneConsts.KEY, this.keyName);
auditMap.put(OzoneConsts.DATA_SIZE, String.valueOf(this.dataSize));
auditMap.put(OzoneConsts.REPLICATION_TYPE,
(this.type != null) ? this.type.name() : null);
auditMap.put(OzoneConsts.REPLICATION_FACTOR,
(this.factor != null) ? this.factor.name() : null);
auditMap.put(OzoneConsts.KEY_LOCATION_INFO,
(this.locationInfoList != null) ? locationInfoList.toString() : null);
return auditMap;
}
/** /**
* Builder class of OmKeyArgs. * Builder class of OmKeyArgs.
*/ */

View File

@ -18,6 +18,8 @@
package org.apache.hadoop.ozone.om.helpers; package org.apache.hadoop.ozone.om.helpers;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.Auditable;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneAclInfo; .OzoneManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto
@ -26,6 +28,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.KeyValue;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -35,7 +38,7 @@ import java.util.stream.Collectors;
/** /**
* A class that encapsulates the OmVolumeArgs Args. * A class that encapsulates the OmVolumeArgs Args.
*/ */
public final class OmVolumeArgs { public final class OmVolumeArgs implements Auditable{
private final String adminName; private final String adminName;
private final String ownerName; private final String ownerName;
private final String volume; private final String volume;
@ -122,6 +125,17 @@ public final class OmVolumeArgs {
return new Builder(); return new Builder();
} }
@Override
public Map<String, String> toAuditMap() {
Map<String, String> auditMap = new LinkedHashMap<>();
auditMap.put(OzoneConsts.ADMIN, this.adminName);
auditMap.put(OzoneConsts.OWNER, this.ownerName);
auditMap.put(OzoneConsts.VOLUME, this.volume);
auditMap.put(OzoneConsts.CREATION_TIME, String.valueOf(this.creationTime));
auditMap.put(OzoneConsts.QUOTA_IN_BYTES, String.valueOf(this.quotaInBytes));
return auditMap;
}
/** /**
* Builder for OmVolumeArgs. * Builder for OmVolumeArgs.
*/ */

View File

@ -36,9 +36,17 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ipc.Client; import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.AuditAction;
import org.apache.hadoop.ozone.audit.AuditEventStatus;
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.AuditLoggerType;
import org.apache.hadoop.ozone.audit.AuditMessage;
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.common.Storage.StorageState; import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@ -58,6 +66,7 @@ import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslat
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
import org.apache.logging.log4j.Level;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -67,6 +76,7 @@ import java.io.PrintStream;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -92,6 +102,9 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private static final Logger LOG = private static final Logger LOG =
LoggerFactory.getLogger(OzoneManager.class); LoggerFactory.getLogger(OzoneManager.class);
private static final AuditLogger AUDIT =
new AuditLogger(AuditLoggerType.OMLOGGER);
private static final String USAGE = private static final String USAGE =
"Usage: \n ozone om [genericOptions] " + "[ " "Usage: \n ozone om [genericOptions] " + "[ "
+ StartupOption.CREATEOBJECTSTORE.getName() + " ]\n " + "ozone om [ " + StartupOption.CREATEOBJECTSTORE.getName() + " ]\n " + "ozone om [ "
@ -454,8 +467,13 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
try { try {
metrics.incNumVolumeCreates(); metrics.incNumVolumeCreates();
volumeManager.createVolume(args); volumeManager.createVolume(args);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_VOLUME,
(args == null) ? null : args.toAuditMap()));
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumVolumeCreateFails(); metrics.incNumVolumeCreateFails();
AUDIT.logWriteFailure(Level.ERROR,
buildAuditMessageForFailure(OMAction.CREATE_VOLUME,
(args == null) ? null : args.toAuditMap()), ex);
throw ex; throw ex;
} }
} }
@ -469,11 +487,17 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
*/ */
@Override @Override
public void setOwner(String volume, String owner) throws IOException { public void setOwner(String volume, String owner) throws IOException {
Map<String, String> auditMap = buildAuditMap(volume);
auditMap.put(OzoneConsts.OWNER, owner);
try { try {
metrics.incNumVolumeUpdates(); metrics.incNumVolumeUpdates();
volumeManager.setOwner(volume, owner); volumeManager.setOwner(volume, owner);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.SET_OWNER,
auditMap));
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumVolumeUpdateFails(); metrics.incNumVolumeUpdateFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.SET_OWNER,
auditMap), ex);
throw ex; throw ex;
} }
} }
@ -487,11 +511,17 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
*/ */
@Override @Override
public void setQuota(String volume, long quota) throws IOException { public void setQuota(String volume, long quota) throws IOException {
Map<String, String> auditMap = buildAuditMap(volume);
auditMap.put(OzoneConsts.QUOTA, String.valueOf(quota));
try { try {
metrics.incNumVolumeUpdates(); metrics.incNumVolumeUpdates();
volumeManager.setQuota(volume, quota); volumeManager.setQuota(volume, quota);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.SET_QUOTA,
auditMap));
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumVolumeUpdateFails(); metrics.incNumVolumeUpdateFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.SET_QUOTA,
auditMap), ex);
throw ex; throw ex;
} }
} }
@ -508,12 +538,24 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
@Override @Override
public boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl) public boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
throws IOException { throws IOException {
boolean auditSuccess = true;
Map<String, String> auditMap = buildAuditMap(volume);
auditMap.put(OzoneConsts.USER_ACL,
(userAcl == null) ? null : userAcl.getName());
try { try {
metrics.incNumVolumeCheckAccesses(); metrics.incNumVolumeCheckAccesses();
return volumeManager.checkVolumeAccess(volume, userAcl); return volumeManager.checkVolumeAccess(volume, userAcl);
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumVolumeCheckAccessFails(); metrics.incNumVolumeCheckAccessFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(
OMAction.CHECK_VOLUME_ACCESS, auditMap), ex);
throw ex; throw ex;
} finally {
if(auditSuccess){
AUDIT.logReadSuccess(buildAuditMessageForSuccess(
OMAction.CHECK_VOLUME_ACCESS, auditMap));
}
} }
} }
@ -526,12 +568,22 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
*/ */
@Override @Override
public OmVolumeArgs getVolumeInfo(String volume) throws IOException { public OmVolumeArgs getVolumeInfo(String volume) throws IOException {
boolean auditSuccess = true;
Map<String, String> auditMap = buildAuditMap(volume);
try { try {
metrics.incNumVolumeInfos(); metrics.incNumVolumeInfos();
return volumeManager.getVolumeInfo(volume); return volumeManager.getVolumeInfo(volume);
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumVolumeInfoFails(); metrics.incNumVolumeInfoFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.READ_VOLUME,
auditMap), ex);
throw ex; throw ex;
} finally {
if(auditSuccess){
AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction.READ_VOLUME,
auditMap));
}
} }
} }
@ -546,8 +598,12 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
try { try {
metrics.incNumVolumeDeletes(); metrics.incNumVolumeDeletes();
volumeManager.deleteVolume(volume); volumeManager.deleteVolume(volume);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.DELETE_VOLUME,
buildAuditMap(volume)));
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumVolumeDeleteFails(); metrics.incNumVolumeDeleteFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_VOLUME,
buildAuditMap(volume)), ex);
throw ex; throw ex;
} }
} }
@ -566,12 +622,26 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
@Override @Override
public List<OmVolumeArgs> listVolumeByUser(String userName, String prefix, public List<OmVolumeArgs> listVolumeByUser(String userName, String prefix,
String prevKey, int maxKeys) throws IOException { String prevKey, int maxKeys) throws IOException {
boolean auditSuccess = true;
Map<String, String> auditMap = new LinkedHashMap<>();
auditMap.put(OzoneConsts.PREV_KEY, prevKey);
auditMap.put(OzoneConsts.PREFIX, prefix);
auditMap.put(OzoneConsts.MAX_KEYS, String.valueOf(maxKeys));
auditMap.put(OzoneConsts.USERNAME, userName);
try { try {
metrics.incNumVolumeLists(); metrics.incNumVolumeLists();
return volumeManager.listVolumes(userName, prefix, prevKey, maxKeys); return volumeManager.listVolumes(userName, prefix, prevKey, maxKeys);
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumVolumeListFails(); metrics.incNumVolumeListFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_VOLUMES,
auditMap), ex);
throw ex; throw ex;
} finally {
if(auditSuccess){
AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_VOLUMES,
auditMap));
}
} }
} }
@ -588,12 +658,26 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
@Override @Override
public List<OmVolumeArgs> listAllVolumes(String prefix, String prevKey, int public List<OmVolumeArgs> listAllVolumes(String prefix, String prevKey, int
maxKeys) throws IOException { maxKeys) throws IOException {
boolean auditSuccess = true;
Map<String, String> auditMap = new LinkedHashMap<>();
auditMap.put(OzoneConsts.PREV_KEY, prevKey);
auditMap.put(OzoneConsts.PREFIX, prefix);
auditMap.put(OzoneConsts.MAX_KEYS, String.valueOf(maxKeys));
auditMap.put(OzoneConsts.USERNAME, null);
try { try {
metrics.incNumVolumeLists(); metrics.incNumVolumeLists();
return volumeManager.listVolumes(null, prefix, prevKey, maxKeys); return volumeManager.listVolumes(null, prefix, prevKey, maxKeys);
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumVolumeListFails(); metrics.incNumVolumeListFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_VOLUMES,
auditMap), ex);
throw ex; throw ex;
} finally {
if(auditSuccess){
AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_VOLUMES,
auditMap));
}
} }
} }
@ -608,8 +692,12 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
try { try {
metrics.incNumBucketCreates(); metrics.incNumBucketCreates();
bucketManager.createBucket(bucketInfo); bucketManager.createBucket(bucketInfo);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_BUCKET,
(bucketInfo == null) ? null : bucketInfo.toAuditMap()));
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumBucketCreateFails(); metrics.incNumBucketCreateFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.CREATE_BUCKET,
(bucketInfo == null) ? null : bucketInfo.toAuditMap()), ex);
throw ex; throw ex;
} }
} }
@ -621,13 +709,27 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
public List<OmBucketInfo> listBuckets(String volumeName, public List<OmBucketInfo> listBuckets(String volumeName,
String startKey, String prefix, int maxNumOfBuckets) String startKey, String prefix, int maxNumOfBuckets)
throws IOException { throws IOException {
boolean auditSuccess = true;
Map<String, String> auditMap = buildAuditMap(volumeName);
auditMap.put(OzoneConsts.START_KEY, startKey);
auditMap.put(OzoneConsts.PREFIX, prefix);
auditMap.put(OzoneConsts.MAX_NUM_OF_BUCKETS,
String.valueOf(maxNumOfBuckets));
try { try {
metrics.incNumBucketLists(); metrics.incNumBucketLists();
return bucketManager.listBuckets(volumeName, return bucketManager.listBuckets(volumeName,
startKey, prefix, maxNumOfBuckets); startKey, prefix, maxNumOfBuckets);
} catch (IOException ex) { } catch (IOException ex) {
metrics.incNumBucketListFails(); metrics.incNumBucketListFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_BUCKETS,
auditMap), ex);
throw ex; throw ex;
} finally {
if(auditSuccess){
AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_BUCKETS,
auditMap));
}
} }
} }
@ -642,12 +744,23 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
@Override @Override
public OmBucketInfo getBucketInfo(String volume, String bucket) public OmBucketInfo getBucketInfo(String volume, String bucket)
throws IOException { throws IOException {
boolean auditSuccess = true;
Map<String, String> auditMap = buildAuditMap(volume);
auditMap.put(OzoneConsts.BUCKET, bucket);
try { try {
metrics.incNumBucketInfos(); metrics.incNumBucketInfos();
return bucketManager.getBucketInfo(volume, bucket); return bucketManager.getBucketInfo(volume, bucket);
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumBucketInfoFails(); metrics.incNumBucketInfoFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.READ_BUCKET,
auditMap), ex);
throw ex; throw ex;
} finally {
if(auditSuccess){
AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction.READ_BUCKET,
auditMap));
}
} }
} }
@ -660,23 +773,39 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
*/ */
@Override @Override
public OpenKeySession openKey(OmKeyArgs args) throws IOException { public OpenKeySession openKey(OmKeyArgs args) throws IOException {
boolean auditSuccess = true;
try { try {
metrics.incNumKeyAllocates(); metrics.incNumKeyAllocates();
return keyManager.openKey(args); return keyManager.openKey(args);
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumKeyAllocateFails(); metrics.incNumKeyAllocateFails();
auditSuccess = false;
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.ALLOCATE_KEY,
(args == null) ? null : args.toAuditMap()), ex);
throw ex; throw ex;
} finally {
if(auditSuccess){
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(
OMAction.ALLOCATE_KEY, (args == null) ? null : args.toAuditMap()));
}
} }
} }
@Override @Override
public void commitKey(OmKeyArgs args, long clientID) public void commitKey(OmKeyArgs args, long clientID)
throws IOException { throws IOException {
Map<String, String> auditMap = (args == null) ? new LinkedHashMap<>() :
args.toAuditMap();
auditMap.put(OzoneConsts.CLIENT_ID, String.valueOf(clientID));
try { try {
metrics.incNumKeyCommits(); metrics.incNumKeyCommits();
keyManager.commitKey(args, clientID); keyManager.commitKey(args, clientID);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.COMMIT_KEY,
auditMap));
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumKeyCommitFails(); metrics.incNumKeyCommitFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.COMMIT_KEY,
auditMap), ex);
throw ex; throw ex;
} }
} }
@ -684,12 +813,24 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
@Override @Override
public OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID) public OmKeyLocationInfo allocateBlock(OmKeyArgs args, long clientID)
throws IOException { throws IOException {
boolean auditSuccess = true;
Map<String, String> auditMap = (args == null) ? new LinkedHashMap<>() :
args.toAuditMap();
auditMap.put(OzoneConsts.CLIENT_ID, String.valueOf(clientID));
try { try {
metrics.incNumBlockAllocateCalls(); metrics.incNumBlockAllocateCalls();
return keyManager.allocateBlock(args, clientID); return keyManager.allocateBlock(args, clientID);
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumBlockAllocateCallFails(); metrics.incNumBlockAllocateCallFails();
auditSuccess = false;
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.ALLOCATE_BLOCK,
auditMap), ex);
throw ex; throw ex;
} finally {
if(auditSuccess){
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(
OMAction.ALLOCATE_BLOCK, auditMap));
}
} }
} }
@ -702,22 +843,38 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
*/ */
@Override @Override
public OmKeyInfo lookupKey(OmKeyArgs args) throws IOException { public OmKeyInfo lookupKey(OmKeyArgs args) throws IOException {
boolean auditSuccess = true;
try { try {
metrics.incNumKeyLookups(); metrics.incNumKeyLookups();
return keyManager.lookupKey(args); return keyManager.lookupKey(args);
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumKeyLookupFails(); metrics.incNumKeyLookupFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.READ_KEY,
(args == null) ? null : args.toAuditMap()), ex);
throw ex; throw ex;
} finally {
if(auditSuccess){
AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction.READ_KEY,
(args == null) ? null : args.toAuditMap()));
}
} }
} }
@Override @Override
public void renameKey(OmKeyArgs args, String toKeyName) throws IOException { public void renameKey(OmKeyArgs args, String toKeyName) throws IOException {
Map<String, String> auditMap = (args == null) ? new LinkedHashMap<>() :
args.toAuditMap();
auditMap.put(OzoneConsts.TO_KEY_NAME, toKeyName);
try { try {
metrics.incNumKeyRenames(); metrics.incNumKeyRenames();
keyManager.renameKey(args, toKeyName); keyManager.renameKey(args, toKeyName);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.RENAME_KEY,
auditMap));
} catch (IOException e) { } catch (IOException e) {
metrics.incNumKeyRenameFails(); metrics.incNumKeyRenameFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.RENAME_KEY,
auditMap), e);
throw e; throw e;
} }
} }
@ -733,8 +890,12 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
try { try {
metrics.incNumKeyDeletes(); metrics.incNumKeyDeletes();
keyManager.deleteKey(args); keyManager.deleteKey(args);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.DELETE_KEY,
(args == null) ? null : args.toAuditMap()));
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumKeyDeleteFails(); metrics.incNumKeyDeleteFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_KEY,
(args == null) ? null : args.toAuditMap()), ex);
throw ex; throw ex;
} }
} }
@ -742,13 +903,27 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
@Override @Override
public List<OmKeyInfo> listKeys(String volumeName, String bucketName, public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
String startKey, String keyPrefix, int maxKeys) throws IOException { String startKey, String keyPrefix, int maxKeys) throws IOException {
boolean auditSuccess = true;
Map<String, String> auditMap = buildAuditMap(volumeName);
auditMap.put(OzoneConsts.BUCKET, bucketName);
auditMap.put(OzoneConsts.START_KEY, startKey);
auditMap.put(OzoneConsts.MAX_KEYS, String.valueOf(maxKeys));
auditMap.put(OzoneConsts.KEY_PREFIX, keyPrefix);
try { try {
metrics.incNumKeyLists(); metrics.incNumKeyLists();
return keyManager.listKeys(volumeName, bucketName, return keyManager.listKeys(volumeName, bucketName,
startKey, keyPrefix, maxKeys); startKey, keyPrefix, maxKeys);
} catch (IOException ex) { } catch (IOException ex) {
metrics.incNumKeyListFails(); metrics.incNumKeyListFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_KEYS,
auditMap), ex);
throw ex; throw ex;
} finally {
if(auditSuccess){
AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction.LIST_KEYS,
auditMap));
}
} }
} }
@ -764,8 +939,12 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
try { try {
metrics.incNumBucketUpdates(); metrics.incNumBucketUpdates();
bucketManager.setBucketProperty(args); bucketManager.setBucketProperty(args);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.UPDATE_BUCKET,
(args == null) ? null : args.toAuditMap()));
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumBucketUpdateFails(); metrics.incNumBucketUpdateFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.UPDATE_BUCKET,
(args == null) ? null : args.toAuditMap()), ex);
throw ex; throw ex;
} }
} }
@ -778,15 +957,49 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* @throws IOException * @throws IOException
*/ */
public void deleteBucket(String volume, String bucket) throws IOException { public void deleteBucket(String volume, String bucket) throws IOException {
Map<String, String> auditMap = buildAuditMap(volume);
auditMap.put(OzoneConsts.BUCKET, bucket);
try { try {
metrics.incNumBucketDeletes(); metrics.incNumBucketDeletes();
bucketManager.deleteBucket(volume, bucket); bucketManager.deleteBucket(volume, bucket);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.DELETE_BUCKET,
auditMap));
} catch (Exception ex) { } catch (Exception ex) {
metrics.incNumBucketDeleteFails(); metrics.incNumBucketDeleteFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_BUCKET,
auditMap), ex);
throw ex; throw ex;
} }
} }
private Map<String, String> buildAuditMap(String volume){
Map<String, String> auditMap = new LinkedHashMap<>();
auditMap.put(OzoneConsts.VOLUME, volume);
return auditMap;
}
// TODO: Temporary method until AuditMessage is simplified
private AuditMessage buildAuditMessageForSuccess(AuditAction op,
Map<String, String> auditMap) {
return new AuditMessage(
(Server.getRemoteUser() == null) ? null :
Server.getRemoteUser().getUserName(),
(Server.getRemoteIp() == null) ? null :
Server.getRemoteIp().getHostAddress(), op.toString(), auditMap,
AuditEventStatus.SUCCESS.toString());
}
// TODO: Temporary method until AuditMessage is simplified
private AuditMessage buildAuditMessageForFailure(AuditAction op,
Map<String, String> auditMap) {
return new AuditMessage(
(Server.getRemoteUser() == null) ? null :
Server.getRemoteUser().getUserName(),
(Server.getRemoteIp() == null) ? null :
Server.getRemoteIp().getHostAddress(), op.toString(), auditMap,
AuditEventStatus.FAILURE.toString());
}
private void registerMXBean() { private void registerMXBean() {
Map<String, String> jmxProperties = new HashMap<String, String>(); Map<String, String> jmxProperties = new HashMap<String, String>();
jmxProperties.put("component", "ServerRuntime"); jmxProperties.put("component", "ServerRuntime");