Adding new files for MAPREDUCE-2655.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1165950 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2011-09-07 01:26:21 +00:00
parent cdfabf5ae2
commit 8815bdf304
4 changed files with 981 additions and 0 deletions

View File

@ -0,0 +1,201 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.nodemanager;
import java.net.InetAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
/**
* Manages NodeManager audit logs.
*
* Audit log format is written as key=value pairs. Tab separated.
*/
public class NMAuditLogger {
private static final Log LOG = LogFactory.getLog(NMAuditLogger.class);
static enum Keys {USER, OPERATION, TARGET, RESULT, IP,
DESCRIPTION, APPID, CONTAINERID}
public static class AuditConstants {
static final String SUCCESS = "SUCCESS";
static final String FAILURE = "FAILURE";
static final String KEY_VAL_SEPARATOR = "=";
static final char PAIR_SEPARATOR = '\t';
// Some commonly used descriptions
public static final String START_CONTAINER = "Start Container Request";
public static final String STOP_CONTAINER = "Stop Container Request";
public static final String FINISH_SUCCESS_CONTAINER = "Container Finished - Succeeded";
public static final String FINISH_FAILED_CONTAINER = "Container Finished - Failed";
public static final String FINISH_KILLED_CONTAINER = "Container Finished - Killed";
}
/**
* A helper api for creating an audit log for a successful event.
*/
static String createSuccessLog(String user, String operation, String target,
ApplicationId appId, ContainerId containerId) {
StringBuilder b = new StringBuilder();
start(Keys.USER, user, b);
addRemoteIP(b);
add(Keys.OPERATION, operation, b);
add(Keys.TARGET, target ,b);
add(Keys.RESULT, AuditConstants.SUCCESS, b);
if (appId != null) {
add(Keys.APPID, appId.toString(), b);
}
if (containerId != null) {
add(Keys.CONTAINERID, containerId.toString(), b);
}
return b.toString();
}
/**
* Create a readable and parseable audit log string for a successful event.
*
* @param user User who made the service request.
* @param operation Operation requested by the user
* @param target The target on which the operation is being performed.
* @param appId Application Id in which operation was performed.
* @param containerId Container Id in which operation was performed.
*
* <br><br>
* Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logSuccess(String user, String operation, String target,
ApplicationId appId, ContainerId containerId) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, appId, containerId));
}
}
/**
* Create a readable and parseable audit log string for a successful event.
*
* @param user User who made the service request.
* @param operation Operation requested by the user
* @param target The target on which the operation is being performed.
*
* <br><br>
* Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logSuccess(String user, String operation, String target) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, null, null));
}
}
/**
* A helper api for creating an audit log for a failure event.
* This is factored out for testing purpose.
*/
static String createFailureLog(String user, String operation, String target,
String description, ApplicationId appId, ContainerId containerId) {
StringBuilder b = new StringBuilder();
start(Keys.USER, user, b);
addRemoteIP(b);
add(Keys.OPERATION, operation, b);
add(Keys.TARGET, target ,b);
add(Keys.RESULT, AuditConstants.FAILURE, b);
add(Keys.DESCRIPTION, description, b);
if (appId != null) {
add(Keys.APPID, appId.toString(), b);
}
if (containerId != null) {
add(Keys.CONTAINERID, containerId.toString(), b);
}
return b.toString();
}
/**
* Create a readable and parseable audit log string for a failed event.
*
* @param user User who made the service request.
* @param operation Operation requested by the user.
* @param target The target on which the operation is being performed.
* @param description Some additional information as to why the operation
* failed.
* @param appId ApplicationId in which operation was performed.
* @param containerId Container Id in which operation was performed.
*
* <br><br>
* Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logFailure(String user, String operation, String target,
String description, ApplicationId appId, ContainerId containerId) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, target, description, appId, containerId));
}
}
/**
* Create a readable and parseable audit log string for a failed event.
*
* @param user User who made the service request.
* @param operation Operation requested by the user.
* @param target The target on which the operation is being performed.
* @param description Some additional information as to why the operation
* failed.
*
* <br><br>
* Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logFailure(String user, String operation,
String target, String description) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, target, description, null, null));
}
}
/**
* A helper api to add remote IP address
*/
static void addRemoteIP(StringBuilder b) {
InetAddress ip = Server.getRemoteIp();
// ip address can be null for testcases
if (ip != null) {
add(Keys.IP, ip.getHostAddress(), b);
}
}
/**
* Adds the first key-val pair to the passed builder in the following format
* key=value
*/
static void start(Keys key, String value, StringBuilder b) {
b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
}
/**
* Appends the key-val pair to the passed builder in the following format
* <pair-delim>key=value
*/
static void add(Keys key, String value, StringBuilder b) {
b.append(AuditConstants.PAIR_SEPARATOR).append(key.name())
.append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
}
}

View File

@ -0,0 +1,227 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.nodemanager;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.TestRPC.TestImpl;
import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.Keys;
import org.apache.hadoop.net.NetUtils;
import static org.mockito.Mockito.*;
import static junit.framework.Assert.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
/**
* Tests {@link NMAuditLogger}.
*/
public class TestNMAuditLogger {
private static final Log LOG = LogFactory.getLog(TestNMAuditLogger.class);
private static final String USER = "test";
private static final String OPERATION = "oper";
private static final String TARGET = "tgt";
private static final String DESC = "description of an audit log";
private static final ApplicationId APPID = mock(ApplicationId.class);
private static final ContainerId CONTAINERID = mock(ContainerId.class);
@Before
public void setUp() throws Exception {
when(APPID.toString()).thenReturn("app_1");
when(CONTAINERID.toString()).thenReturn("container_1");
}
/**
* Test the AuditLog format with key-val pair.
*/
@Test
public void testKeyValLogFormat() throws Exception {
StringBuilder actLog = new StringBuilder();
StringBuilder expLog = new StringBuilder();
// add the first k=v pair and check
NMAuditLogger.start(Keys.USER, USER, actLog);
expLog.append("USER=test");
assertEquals(expLog.toString(), actLog.toString());
// append another k1=v1 pair to already added k=v and test
NMAuditLogger.add(Keys.OPERATION, OPERATION, actLog);
expLog.append("\tOPERATION=oper");
assertEquals(expLog.toString(), actLog.toString());
// append another k1=null pair and test
NMAuditLogger.add(Keys.APPID, (String)null, actLog);
expLog.append("\tAPPID=null");
assertEquals(expLog.toString(), actLog.toString());
// now add the target and check of the final string
NMAuditLogger.add(Keys.TARGET, TARGET, actLog);
expLog.append("\tTARGET=tgt");
assertEquals(expLog.toString(), actLog.toString());
}
/**
* Test the AuditLog format for successful events.
*/
private void testSuccessLogFormatHelper(boolean checkIP,
ApplicationId appId, ContainerId containerId) {
// check without the IP
String sLog = NMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
appId, containerId);
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
if (checkIP) {
InetAddress ip = Server.getRemoteIp();
expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
}
expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");
if (appId != null) {
expLog.append("\tAPPID=app_1");
}
if (containerId != null) {
expLog.append("\tCONTAINERID=container_1");
}
assertEquals(expLog.toString(), sLog);
}
/**
* Test the AuditLog format for successful events passing nulls.
*/
private void testSuccessLogNulls(boolean checkIP) {
String sLog = NMAuditLogger.createSuccessLog(null, null, null,
null, null);
StringBuilder expLog = new StringBuilder();
expLog.append("USER=null\t");
if (checkIP) {
InetAddress ip = Server.getRemoteIp();
expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
}
expLog.append("OPERATION=null\tTARGET=null\tRESULT=SUCCESS");
assertEquals(expLog.toString(), sLog);
}
/**
* Test the AuditLog format for successful events with the various
* parameters.
*/
private void testSuccessLogFormat(boolean checkIP) {
testSuccessLogFormatHelper(checkIP, null, null);
testSuccessLogFormatHelper(checkIP, APPID, null);
testSuccessLogFormatHelper(checkIP, null, CONTAINERID);
testSuccessLogFormatHelper(checkIP, APPID, CONTAINERID);
testSuccessLogNulls(checkIP);
}
/**
* Test the AuditLog format for failure events.
*/
private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
ContainerId containerId) {
String fLog =
NMAuditLogger.createFailureLog(USER, OPERATION, TARGET, DESC, appId,
containerId);
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
if (checkIP) {
InetAddress ip = Server.getRemoteIp();
expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
}
expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t");
expLog.append("DESCRIPTION=description of an audit log");
if (appId != null) {
expLog.append("\tAPPID=app_1");
}
if (containerId != null) {
expLog.append("\tCONTAINERID=container_1");
}
assertEquals(expLog.toString(), fLog);
}
/**
* Test the AuditLog format for failure events with the various
* parameters.
*/
private void testFailureLogFormat(boolean checkIP) {
testFailureLogFormatHelper(checkIP, null, null);
testFailureLogFormatHelper(checkIP, APPID, null);
testFailureLogFormatHelper(checkIP, null, CONTAINERID);
testFailureLogFormatHelper(checkIP, APPID, CONTAINERID);
}
/**
* Test {@link NMAuditLogger} without IP set.
*/
@Test
public void testNMAuditLoggerWithoutIP() throws Exception {
// test without ip
testSuccessLogFormat(false);
testFailureLogFormat(false);
}
/**
* A special extension of {@link TestImpl} RPC server with
* {@link TestImpl#ping()} testing the audit logs.
*/
private class MyTestRPCServer extends TestImpl {
@Override
public void ping() {
// test with ip set
testSuccessLogFormat(true);
testFailureLogFormat(true);
}
}
/**
* Test {@link NMAuditLogger} with IP set.
*/
@Test
public void testNMAuditLoggerWithIP() throws Exception {
Configuration conf = new Configuration();
// start the IPC server
Server server = RPC.getServer(new MyTestRPCServer(), "0.0.0.0", 0, conf);
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
// Make a client connection and test the audit log
TestProtocol proxy = (TestProtocol)RPC.getProxy(TestProtocol.class,
TestProtocol.versionID, addr, conf);
// Start the testcase
proxy.ping();
server.stop();
}
}

View File

@ -0,0 +1,309 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager;
import java.net.InetAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
/**
* Manages ResourceManager audit logs.
*
* Audit log format is written as key=value pairs. Tab separated.
*/
public class RMAuditLogger {
private static final Log LOG = LogFactory.getLog(RMAuditLogger.class);
static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS,
DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID}
public static class AuditConstants {
static final String SUCCESS = "SUCCESS";
static final String FAILURE = "FAILURE";
static final String KEY_VAL_SEPARATOR = "=";
static final char PAIR_SEPARATOR = '\t';
public static final String KILL_APP_REQUEST = "Kill Application Request";
public static final String SUBMIT_APP_REQUEST = "Submit Application Request";
public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded";
public static final String FINISH_FAILED_APP = "Application Finished - Failed";
public static final String FINISH_KILLED_APP = "Application Finished - Killed";
public static final String REGISTER_AM = "Register App Master";
public static final String ALLOC_CONTAINER = "AM Allocated Container";
public static final String RELEASE_CONTAINER = "AM Released Container";
// Some commonly used descriptions
public static final String UNAUTHORIZED_USER = "Unauthorized user";
}
/**
* A helper api for creating an audit log for a successful event.
*/
static String createSuccessLog(String user, String operation, String target,
ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) {
StringBuilder b = new StringBuilder();
start(Keys.USER, user, b);
addRemoteIP(b);
add(Keys.OPERATION, operation, b);
add(Keys.TARGET, target ,b);
add(Keys.RESULT, AuditConstants.SUCCESS, b);
if (appId != null) {
add(Keys.APPID, appId.toString(), b);
}
if (attemptId != null) {
add(Keys.APPATTEMPTID, attemptId.toString(), b);
}
if (containerId != null) {
add(Keys.CONTAINERID, containerId.toString(), b);
}
return b.toString();
}
/**
* Create a readable and parseable audit log string for a successful event.
*
* @param user User who made the service request to the ResourceManager
* @param operation Operation requested by the user.
* @param target The target on which the operation is being performed.
* @param appId Application Id in which operation was performed.
* @param containerId Container Id in which operation was performed.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logSuccess(String user, String operation, String target,
ApplicationId appId, ContainerId containerId) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, appId, null,
containerId));
}
}
/**
* Create a readable and parseable audit log string for a successful event.
*
* @param user User who made the service request to the ResourceManager.
* @param operation Operation requested by the user.
* @param target The target on which the operation is being performed.
* @param appId Application Id in which operation was performed.
* @param attemptId Application Attempt Id in which operation was performed.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logSuccess(String user, String operation, String target,
ApplicationId appId, ApplicationAttemptId attemptId) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, appId, attemptId,
null));
}
}
/**
* Create a readable and parseable audit log string for a successful event.
*
* @param user User who made the service request to the ResourceManager.
* @param operation Operation requested by the user.
* @param target The target on which the operation is being performed.
* @param appId Application Id in which operation was performed.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logSuccess(String user, String operation, String target,
ApplicationId appId) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, appId, null, null));
}
}
/**
* Create a readable and parseable audit log string for a successful event.
*
* @param user User who made the service request.
* @param operation Operation requested by the user.
* @param target The target on which the operation is being performed.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logSuccess(String user, String operation, String target) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, null, null, null));
}
}
/**
* A helper api for creating an audit log for a failure event.
*/
static String createFailureLog(String user, String operation, String perm,
String target, String description, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId) {
StringBuilder b = new StringBuilder();
start(Keys.USER, user, b);
addRemoteIP(b);
add(Keys.OPERATION, operation, b);
add(Keys.TARGET, target ,b);
add(Keys.RESULT, AuditConstants.FAILURE, b);
add(Keys.DESCRIPTION, description, b);
add(Keys.PERMISSIONS, perm, b);
if (appId != null) {
add(Keys.APPID, appId.toString(), b);
}
if (attemptId != null) {
add(Keys.APPATTEMPTID, attemptId.toString(), b);
}
if (containerId != null) {
add(Keys.CONTAINERID, containerId.toString(), b);
}
return b.toString();
}
/**
* Create a readable and parseable audit log string for a failed event.
*
* @param user User who made the service request.
* @param operation Operation requested by the user.
* @param perm Target permissions.
* @param target The target on which the operation is being performed.
* @param description Some additional information as to why the operation
* failed.
* @param appId Application Id in which operation was performed.
* @param containerId Container Id in which operation was performed.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logFailure(String user, String operation, String perm,
String target, String description, ApplicationId appId,
ContainerId containerId) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
appId, null, containerId));
}
}
/**
* Create a readable and parseable audit log string for a failed event.
*
* @param user User who made the service request.
* @param operation Operation requested by the user.
* @param perm Target permissions.
* @param target The target on which the operation is being performed.
* @param description Some additional information as to why the operation
* failed.
* @param appId ApplicationId in which operation was performed.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logFailure(String user, String operation, String perm,
String target, String description, ApplicationId appId,
ApplicationAttemptId attemptId) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
appId, attemptId, null));
}
}
/**
* Create a readable and parseable audit log string for a failed event.
*
* @param user User who made the service request.
* @param operation Operation requested by the user.
* @param perm Target permissions.
* @param target The target on which the operation is being performed.
* @param description Some additional information as to why the operation
* failed.
* @param appId ApplicationId in which operation was performed.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logFailure(String user, String operation, String perm,
String target, String description, ApplicationId appId) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
appId, null, null));
}
}
/**
* Create a readable and parseable audit log string for a failed event.
*
* @param user User who made the service request.
* @param operation Operation requested by the user.
* @param perm Target permissions.
* @param target The target on which the operation is being performed.
* @param description Some additional information as to why the operation
* failed.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logFailure(String user, String operation, String perm,
String target, String description) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
null, null, null));
}
}
/**
* A helper api to add remote IP address
*/
static void addRemoteIP(StringBuilder b) {
InetAddress ip = Server.getRemoteIp();
// ip address can be null for testcases
if (ip != null) {
add(Keys.IP, ip.getHostAddress(), b);
}
}
/**
* Adds the first key-val pair to the passed builder in the following format
* key=value
*/
static void start(Keys key, String value, StringBuilder b) {
b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
}
/**
* Appends the key-val pair to the passed builder in the following format
* <pair-delim>key=value
*/
static void add(Keys key, String value, StringBuilder b) {
b.append(AuditConstants.PAIR_SEPARATOR).append(key.name())
.append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
}
}

View File

@ -0,0 +1,244 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.TestRPC.TestImpl;
import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
import org.apache.hadoop.net.NetUtils;
import static org.mockito.Mockito.*;
import static junit.framework.Assert.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
/**
* Tests {@link RMAuditLogger}.
*/
public class TestRMAuditLogger {
private static final Log LOG = LogFactory.getLog(TestRMAuditLogger.class);
private static final String USER = "test";
private static final String OPERATION = "oper";
private static final String TARGET = "tgt";
private static final String PERM = "admin group";
private static final String DESC = "description of an audit log";
private static final ApplicationId APPID = mock(ApplicationId.class);
private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class);
private static final ContainerId CONTAINERID = mock(ContainerId.class);
@Before
public void setUp() throws Exception {
when(APPID.toString()).thenReturn("app_1");
when(ATTEMPTID.toString()).thenReturn("app_attempt_1");
when(CONTAINERID.toString()).thenReturn("container_1");
}
/**
* Test the AuditLog format with key-val pair.
*/
@Test
public void testKeyValLogFormat() throws Exception {
StringBuilder actLog = new StringBuilder();
StringBuilder expLog = new StringBuilder();
// add the first k=v pair and check
RMAuditLogger.start(Keys.USER, USER, actLog);
expLog.append("USER=test");
assertEquals(expLog.toString(), actLog.toString());
// append another k1=v1 pair to already added k=v and test
RMAuditLogger.add(Keys.OPERATION, OPERATION, actLog);
expLog.append("\tOPERATION=oper");
assertEquals(expLog.toString(), actLog.toString());
// append another k1=null pair and test
RMAuditLogger.add(Keys.APPID, (String)null, actLog);
expLog.append("\tAPPID=null");
assertEquals(expLog.toString(), actLog.toString());
// now add the target and check of the final string
RMAuditLogger.add(Keys.TARGET, TARGET, actLog);
expLog.append("\tTARGET=tgt");
assertEquals(expLog.toString(), actLog.toString());
}
/**
* Test the AuditLog format for successful events.
*/
private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId) {
String sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
appId, attemptId, containerId);
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
if (checkIP) {
InetAddress ip = Server.getRemoteIp();
expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
}
expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");
if (appId != null) {
expLog.append("\tAPPID=app_1");
}
if (attemptId != null) {
expLog.append("\tAPPATTEMPTID=app_attempt_1");
}
if (containerId != null) {
expLog.append("\tCONTAINERID=container_1");
}
assertEquals(expLog.toString(), sLog);
}
/**
* Test the AuditLog format for successful events passing nulls.
*/
private void testSuccessLogNulls(boolean checkIP) {
String sLog = RMAuditLogger.createSuccessLog(null, null, null, null,
null, null);
StringBuilder expLog = new StringBuilder();
expLog.append("USER=null\t");
if (checkIP) {
InetAddress ip = Server.getRemoteIp();
expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
}
expLog.append("OPERATION=null\tTARGET=null\tRESULT=SUCCESS");
assertEquals(expLog.toString(), sLog);
}
/**
* Test the AuditLog format for successful events with the various
* parameters.
*/
private void testSuccessLogFormat(boolean checkIP) {
testSuccessLogFormatHelper(checkIP, null, null, null);
testSuccessLogFormatHelper(checkIP, APPID, null, null);
testSuccessLogFormatHelper(checkIP, null, null, CONTAINERID);
testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, null);
testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, null);
testSuccessLogFormatHelper(checkIP, APPID, null, CONTAINERID);
testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID);
testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID);
testSuccessLogNulls(checkIP);
}
/**
* Test the AuditLog format for failure events.
*/
private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId) {
String fLog =
RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
appId, attemptId, containerId);
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
if (checkIP) {
InetAddress ip = Server.getRemoteIp();
expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
}
expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t");
expLog.append("DESCRIPTION=description of an audit log");
expLog.append("\tPERMISSIONS=admin group");
if (appId != null) {
expLog.append("\tAPPID=app_1");
}
if (attemptId != null) {
expLog.append("\tAPPATTEMPTID=app_attempt_1");
}
if (containerId != null) {
expLog.append("\tCONTAINERID=container_1");
}
assertEquals(expLog.toString(), fLog);
}
/**
* Test the AuditLog format for failure events with the various
* parameters.
*/
private void testFailureLogFormat(boolean checkIP) {
testFailureLogFormatHelper(checkIP, null, null, null);
testFailureLogFormatHelper(checkIP, APPID, null, null);
testFailureLogFormatHelper(checkIP, null, null, CONTAINERID);
testFailureLogFormatHelper(checkIP, null, ATTEMPTID, null);
testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, null);
testFailureLogFormatHelper(checkIP, APPID, null, CONTAINERID);
testFailureLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID);
testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID);
}
/**
* Test {@link RMAuditLogger} without IP set.
*/
@Test
public void testRMAuditLoggerWithoutIP() throws Exception {
// test without ip
testSuccessLogFormat(false);
testFailureLogFormat(false);
}
/**
* A special extension of {@link TestImpl} RPC server with
* {@link TestImpl#ping()} testing the audit logs.
*/
private class MyTestRPCServer extends TestImpl {
@Override
public void ping() {
// test with ip set
testSuccessLogFormat(true);
testFailureLogFormat(true);
}
}
/**
* Test {@link RMAuditLogger} with IP set.
*/
@Test
public void testRMAuditLoggerWithIP() throws Exception {
Configuration conf = new Configuration();
// start the IPC server
Server server = RPC.getServer(new MyTestRPCServer(), "0.0.0.0", 0, conf);
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
// Make a client connection and test the audit log
TestProtocol proxy = (TestProtocol)RPC.getProxy(TestProtocol.class,
TestProtocol.versionID, addr, conf);
// Start the testcase
proxy.ping();
server.stop();
}
}