Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-14296. Move logging APIs over to slf4j in hadoop-tools.

This commit is contained in:
Akira Ajisaka 2017-06-20 13:21:12 +09:00
parent b19ca498ae
commit 8a77cf1ee4
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
8 changed files with 90 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java
View File

@ -27,18 +27,18 @@ import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem.FolderRenamePending; import org.apache.hadoop.fs.azure.NativeAzureFileSystem.FolderRenamePending;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer; import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.log4j.Logger;
import org.junit.Before; import org.junit.Before;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* Tests the Native Azure file system (WASB) using parallel threads for rename and delete operations. * Tests the Native Azure file system (WASB) using parallel threads for rename and delete operations.
@ -68,8 +68,8 @@ public class TestFileSystemOperationsWithThreads extends AbstractWasbTestBase {
fs.initialize(uri, conf); fs.initialize(uri, conf);
// Capture logs // Capture logs
logs = LogCapturer.captureLogs(new Log4JLogger(Logger logs = LogCapturer.captureLogs(
.getRootLogger())); LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME));
} }
/* /*

12
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemClientLogging.java
View File

@ -24,12 +24,12 @@ import static org.junit.Assert.assertTrue;
import java.net.URI; import java.net.URI;
import java.util.StringTokenizer; import java.util.StringTokenizer;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer; import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.log4j.Logger;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* Test to validate Azure storage client side logging. Tests works only when * Test to validate Azure storage client side logging. Tests works only when
@ -97,8 +97,8 @@ public class TestNativeAzureFileSystemClientLogging
@Test @Test
public void testLoggingEnabled() throws Exception { public void testLoggingEnabled() throws Exception {
LogCapturer logs = LogCapturer.captureLogs(new Log4JLogger(Logger LogCapturer logs = LogCapturer.captureLogs(
.getRootLogger())); LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME));
// Update configuration based on the Test. // Update configuration based on the Test.
updateFileSystemConfiguration(true); updateFileSystemConfiguration(true);
@ -111,8 +111,8 @@ public class TestNativeAzureFileSystemClientLogging
@Test @Test
public void testLoggingDisabled() throws Exception { public void testLoggingDisabled() throws Exception {
LogCapturer logs = LogCapturer.captureLogs(new Log4JLogger(Logger LogCapturer logs = LogCapturer.captureLogs(
.getRootLogger())); LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME));
// Update configuration based on the Test. // Update configuration based on the Test.
updateFileSystemConfiguration(false); updateFileSystemConfiguration(false);

39
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
View File

@ -22,7 +22,6 @@ import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.Reader; import java.io.Reader;
import java.text.MessageFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
@ -65,9 +64,10 @@ import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper; import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper;
import org.apache.hadoop.yarn.sls.utils.SLSUtils; import org.apache.hadoop.yarn.sls.utils.SLSUtils;
import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.log4j.Logger;
import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Private @Private
@Unstable @Unstable
@ -102,7 +102,7 @@ public class SLSRunner {
new HashMap<String, Object>(); new HashMap<String, Object>();
// logger // logger
public final static Logger LOG = Logger.getLogger(SLSRunner.class); public final static Logger LOG = LoggerFactory.getLogger(SLSRunner.class);
// input traces, input-rumen or input-sls // input traces, input-rumen or input-sls
private boolean isSLS; private boolean isSLS;
@ -236,13 +236,12 @@ public class SLSRunner {
if (numRunningNodes == numNMs) { if (numRunningNodes == numNMs) {
break; break;
} }
LOG.info(MessageFormat.format("SLSRunner is waiting for all " + LOG.info("SLSRunner is waiting for all nodes RUNNING."
"nodes RUNNING. {0} of {1} NMs initialized.", + " {} of {} NMs initialized.", numRunningNodes, numNMs);
numRunningNodes, numNMs));
Thread.sleep(1000); Thread.sleep(1000);
} }
LOG.info(MessageFormat.format("SLSRunner takes {0} ms to launch all nodes.", LOG.info("SLSRunner takes {} ms to launch all nodes.",
(System.currentTimeMillis() - startTimeMS))); System.currentTimeMillis() - startTimeMS);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@ -390,7 +389,7 @@ public class SLSRunner {
jobStartTimeMS -= baselineTimeMS; jobStartTimeMS -= baselineTimeMS;
jobFinishTimeMS -= baselineTimeMS; jobFinishTimeMS -= baselineTimeMS;
if (jobStartTimeMS < 0) { if (jobStartTimeMS < 0) {
LOG.warn("Warning: reset job " + oldJobId + " start time to 0."); LOG.warn("Warning: reset job {} start time to 0.", oldJobId);
jobFinishTimeMS = jobFinishTimeMS - jobStartTimeMS; jobFinishTimeMS = jobFinishTimeMS - jobStartTimeMS;
jobStartTimeMS = 0; jobStartTimeMS = 0;
} }
@ -454,14 +453,14 @@ public class SLSRunner {
if (printSimulation) { if (printSimulation) {
// node // node
LOG.info("------------------------------------"); LOG.info("------------------------------------");
LOG.info(MessageFormat.format("# nodes = {0}, # racks = {1}, capacity " + LOG.info("# nodes = {}, # racks = {}, capacity " +
"of each node {2} MB memory and {3} vcores.", "of each node {} MB memory and {} vcores.",
numNMs, numRacks, nmMemoryMB, nmVCores)); numNMs, numRacks, nmMemoryMB, nmVCores);
LOG.info("------------------------------------"); LOG.info("------------------------------------");
// job // job
LOG.info(MessageFormat.format("# applications = {0}, # total " + LOG.info("# applications = {}, # total " +
"tasks = {1}, average # tasks per application = {2}", "tasks = {}, average # tasks per application = {}",
numAMs, numTasks, (int)(Math.ceil((numTasks + 0.0) / numAMs)))); numAMs, numTasks, (int)(Math.ceil((numTasks + 0.0) / numAMs)));
LOG.info("JobId\tQueue\tAMType\tDuration\t#Tasks"); LOG.info("JobId\tQueue\tAMType\tDuration\t#Tasks");
for (Map.Entry<String, AMSimulator> entry : amMap.entrySet()) { for (Map.Entry<String, AMSimulator> entry : amMap.entrySet()) {
AMSimulator am = entry.getValue(); AMSimulator am = entry.getValue();
@ -470,13 +469,13 @@ public class SLSRunner {
} }
LOG.info("------------------------------------"); LOG.info("------------------------------------");
// queue // queue
LOG.info(MessageFormat.format("number of queues = {0} average " + LOG.info("number of queues = {} average number of apps = {}",
"number of apps = {1}", queueAppNumMap.size(), queueAppNumMap.size(),
(int)(Math.ceil((numAMs + 0.0) / queueAppNumMap.size())))); (int)(Math.ceil((numAMs + 0.0) / queueAppNumMap.size())));
LOG.info("------------------------------------"); LOG.info("------------------------------------");
// runtime // runtime
LOG.info(MessageFormat.format("estimated simulation time is {0}" + LOG.info("estimated simulation time is {} seconds",
" seconds", (long)(Math.ceil(maxRuntime / 1000.0)))); (long)(Math.ceil(maxRuntime / 1000.0)));
LOG.info("------------------------------------"); LOG.info("------------------------------------");
} }
// package these information in the simulateInfoMap used by other places // package these information in the simulateInfoMap used by other places

13
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
View File

@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.sls.appmaster;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction; import java.security.PrivilegedExceptionAction;
import java.text.MessageFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -66,13 +65,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Logger;
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator; import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper; import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper;
import org.apache.hadoop.yarn.sls.SLSRunner; import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.hadoop.yarn.sls.scheduler.TaskRunner; import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
import org.apache.hadoop.yarn.sls.utils.SLSUtils; import org.apache.hadoop.yarn.sls.utils.SLSUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Private @Private
@Unstable @Unstable
@ -108,7 +108,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
protected int totalContainers; protected int totalContainers;
protected int finishedContainers; protected int finishedContainers;
protected final Logger LOG = Logger.getLogger(AMSimulator.class); protected final Logger LOG = LoggerFactory.getLogger(AMSimulator.class);
public AMSimulator() { public AMSimulator() {
this.responseQueue = new LinkedBlockingQueue<AllocateResponse>(); this.responseQueue = new LinkedBlockingQueue<AllocateResponse>();
@ -163,7 +163,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
@Override @Override
public void lastStep() throws Exception { public void lastStep() throws Exception {
LOG.info(MessageFormat.format("Application {0} is shutting down.", appId)); LOG.info("Application {} is shutting down.", appId);
// unregister tracking // unregister tracking
if (isTracked) { if (isTracked) {
untrackApp(); untrackApp();
@ -266,7 +266,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
return null; return null;
} }
}); });
LOG.info(MessageFormat.format("Submit a new application {0}", appId)); LOG.info("Submit a new application {}", appId);
// waiting until application ACCEPTED // waiting until application ACCEPTED
RMApp app = rm.getRMContext().getRMApps().get(appId); RMApp app = rm.getRMContext().getRMApps().get(appId);
@ -309,8 +309,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
} }
}); });
LOG.info(MessageFormat.format( LOG.info("Register the application master for application {}", appId);
"Register the application master for application {0}", appId));
} }
private void trackApp() { private void trackApp() {

66
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java
View File

@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.sls.appmaster;
import java.io.IOException; import java.io.IOException;
import java.security.PrivilegedExceptionAction; import java.security.PrivilegedExceptionAction;
import java.text.MessageFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
@ -45,7 +44,8 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator; import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
import org.apache.hadoop.yarn.sls.SLSRunner; import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.log4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Private @Private
@Unstable @Unstable
@ -116,7 +116,7 @@ public class MRAMSimulator extends AMSimulator {
private final static int MR_AM_CONTAINER_RESOURCE_MEMORY_MB = 1024; private final static int MR_AM_CONTAINER_RESOURCE_MEMORY_MB = 1024;
private final static int MR_AM_CONTAINER_RESOURCE_VCORES = 1; private final static int MR_AM_CONTAINER_RESOURCE_VCORES = 1;
public final Logger LOG = Logger.getLogger(MRAMSimulator.class); public final Logger LOG = LoggerFactory.getLogger(MRAMSimulator.class);
public void init(int id, int heartbeatInterval, public void init(int id, int heartbeatInterval,
List<ContainerSimulator> containerList, ResourceManager rm, SLSRunner se, List<ContainerSimulator> containerList, ResourceManager rm, SLSRunner se,
@ -162,8 +162,7 @@ public class MRAMSimulator extends AMSimulator {
MR_AM_CONTAINER_RESOURCE_VCORES), MR_AM_CONTAINER_RESOURCE_VCORES),
ResourceRequest.ANY, 1, 1); ResourceRequest.ANY, 1, 1);
ask.add(amRequest); ask.add(amRequest);
LOG.debug(MessageFormat.format("Application {0} sends out allocate " + LOG.debug("Application {} sends out allocate request for its AM", appId);
"request for its AM", appId));
final AllocateRequest request = this.createAllocateRequest(ask); final AllocateRequest request = this.createAllocateRequest(ask);
UserGroupInformation ugi = UserGroupInformation ugi =
@ -200,8 +199,8 @@ public class MRAMSimulator extends AMSimulator {
.addNewContainer(container, -1L); .addNewContainer(container, -1L);
// Start AM container // Start AM container
amContainer = container; amContainer = container;
LOG.debug(MessageFormat.format("Application {0} starts its " + LOG.debug("Application {} starts its AM container ({}).", appId,
"AM container ({1}).", appId, amContainer.getId())); amContainer.getId());
isAMContainerRunning = true; isAMContainerRunning = true;
} }
} }
@ -217,36 +216,35 @@ public class MRAMSimulator extends AMSimulator {
ContainerId containerId = cs.getContainerId(); ContainerId containerId = cs.getContainerId();
if (cs.getExitStatus() == ContainerExitStatus.SUCCESS) { if (cs.getExitStatus() == ContainerExitStatus.SUCCESS) {
if (assignedMaps.containsKey(containerId)) { if (assignedMaps.containsKey(containerId)) {
LOG.debug(MessageFormat.format("Application {0} has one" + LOG.debug("Application {} has one mapper finished ({}).",
"mapper finished ({1}).", appId, containerId)); appId, containerId);
assignedMaps.remove(containerId); assignedMaps.remove(containerId);
mapFinished ++; mapFinished ++;
finishedContainers ++; finishedContainers ++;
} else if (assignedReduces.containsKey(containerId)) { } else if (assignedReduces.containsKey(containerId)) {
LOG.debug(MessageFormat.format("Application {0} has one" + LOG.debug("Application {} has one reducer finished ({}).",
"reducer finished ({1}).", appId, containerId)); appId, containerId);
assignedReduces.remove(containerId); assignedReduces.remove(containerId);
reduceFinished ++; reduceFinished ++;
finishedContainers ++; finishedContainers ++;
} else { } else {
// am container released event // am container released event
isFinished = true; isFinished = true;
LOG.info(MessageFormat.format("Application {0} goes to " + LOG.info("Application {} goes to finish.", appId);
"finish.", appId));
} }
} else { } else {
// container to be killed // container to be killed
if (assignedMaps.containsKey(containerId)) { if (assignedMaps.containsKey(containerId)) {
LOG.debug(MessageFormat.format("Application {0} has one " + LOG.debug("Application {} has one mapper killed ({}).",
"mapper killed ({1}).", appId, containerId)); appId, containerId);
pendingFailedMaps.add(assignedMaps.remove(containerId)); pendingFailedMaps.add(assignedMaps.remove(containerId));
} else if (assignedReduces.containsKey(containerId)) { } else if (assignedReduces.containsKey(containerId)) {
LOG.debug(MessageFormat.format("Application {0} has one " + LOG.debug("Application {} has one reducer killed ({}).",
"reducer killed ({1}).", appId, containerId)); appId, containerId);
pendingFailedReduces.add(assignedReduces.remove(containerId)); pendingFailedReduces.add(assignedReduces.remove(containerId));
} else { } else {
LOG.info(MessageFormat.format("Application {0}'s AM is " + LOG.info("Application {}'s AM is going to be killed." +
"going to be killed. Restarting...", appId)); " Restarting...", appId);
restart(); restart();
} }
} }
@ -261,8 +259,8 @@ public class MRAMSimulator extends AMSimulator {
se.getNmMap().get(amContainer.getNodeId()) se.getNmMap().get(amContainer.getNodeId())
.cleanupContainer(amContainer.getId()); .cleanupContainer(amContainer.getId());
isAMContainerRunning = false; isAMContainerRunning = false;
LOG.debug(MessageFormat.format("Application {0} sends out event " + LOG.debug("Application {} sends out event to clean up"
"to clean up its AM container.", appId)); + " its AM container.", appId);
isFinished = true; isFinished = true;
break; break;
} }
@ -271,15 +269,15 @@ public class MRAMSimulator extends AMSimulator {
for (Container container : response.getAllocatedContainers()) { for (Container container : response.getAllocatedContainers()) {
if (! scheduledMaps.isEmpty()) { if (! scheduledMaps.isEmpty()) {
ContainerSimulator cs = scheduledMaps.remove(); ContainerSimulator cs = scheduledMaps.remove();
LOG.debug(MessageFormat.format("Application {0} starts a " + LOG.debug("Application {} starts to launch a mapper ({}).",
"launch a mapper ({1}).", appId, container.getId())); appId, container.getId());
assignedMaps.put(container.getId(), cs); assignedMaps.put(container.getId(), cs);
se.getNmMap().get(container.getNodeId()) se.getNmMap().get(container.getNodeId())
.addNewContainer(container, cs.getLifeTime()); .addNewContainer(container, cs.getLifeTime());
} else if (! this.scheduledReduces.isEmpty()) { } else if (! this.scheduledReduces.isEmpty()) {
ContainerSimulator cs = scheduledReduces.remove(); ContainerSimulator cs = scheduledReduces.remove();
LOG.debug(MessageFormat.format("Application {0} starts a " + LOG.debug("Application {} starts to launch a reducer ({}).",
"launch a reducer ({1}).", appId, container.getId())); appId, container.getId());
assignedReduces.put(container.getId(), cs); assignedReduces.put(container.getId(), cs);
se.getNmMap().get(container.getNodeId()) se.getNmMap().get(container.getNodeId())
.addNewContainer(container, cs.getLifeTime()); .addNewContainer(container, cs.getLifeTime());
@ -324,15 +322,14 @@ public class MRAMSimulator extends AMSimulator {
// map phase // map phase
if (! pendingMaps.isEmpty()) { if (! pendingMaps.isEmpty()) {
ask = packageRequests(pendingMaps, PRIORITY_MAP); ask = packageRequests(pendingMaps, PRIORITY_MAP);
LOG.debug(MessageFormat.format("Application {0} sends out " + LOG.debug("Application {} sends out request for {} mappers.",
"request for {1} mappers.", appId, pendingMaps.size())); appId, pendingMaps.size());
scheduledMaps.addAll(pendingMaps); scheduledMaps.addAll(pendingMaps);
pendingMaps.clear(); pendingMaps.clear();
} else if (! pendingFailedMaps.isEmpty() && scheduledMaps.isEmpty()) { } else if (! pendingFailedMaps.isEmpty() && scheduledMaps.isEmpty()) {
ask = packageRequests(pendingFailedMaps, PRIORITY_MAP); ask = packageRequests(pendingFailedMaps, PRIORITY_MAP);
LOG.debug(MessageFormat.format("Application {0} sends out " + LOG.debug("Application {} sends out requests for {} failed mappers.",
"requests for {1} failed mappers.", appId, appId, pendingFailedMaps.size());
pendingFailedMaps.size()));
scheduledMaps.addAll(pendingFailedMaps); scheduledMaps.addAll(pendingFailedMaps);
pendingFailedMaps.clear(); pendingFailedMaps.clear();
} }
@ -340,16 +337,15 @@ public class MRAMSimulator extends AMSimulator {
// reduce phase // reduce phase
if (! pendingReduces.isEmpty()) { if (! pendingReduces.isEmpty()) {
ask = packageRequests(pendingReduces, PRIORITY_REDUCE); ask = packageRequests(pendingReduces, PRIORITY_REDUCE);
LOG.debug(MessageFormat.format("Application {0} sends out " + LOG.debug("Application {} sends out requests for {} reducers.",
"requests for {1} reducers.", appId, pendingReduces.size())); appId, pendingReduces.size());
scheduledReduces.addAll(pendingReduces); scheduledReduces.addAll(pendingReduces);
pendingReduces.clear(); pendingReduces.clear();
} else if (! pendingFailedReduces.isEmpty() } else if (! pendingFailedReduces.isEmpty()
&& scheduledReduces.isEmpty()) { && scheduledReduces.isEmpty()) {
ask = packageRequests(pendingFailedReduces, PRIORITY_REDUCE); ask = packageRequests(pendingFailedReduces, PRIORITY_REDUCE);
LOG.debug(MessageFormat.format("Application {0} sends out " + LOG.debug("Application {} sends out request for {} failed reducers.",
"request for {1} failed reducers.", appId, appId, pendingFailedReduces.size());
pendingFailedReduces.size()));
scheduledReduces.addAll(pendingFailedReduces); scheduledReduces.addAll(pendingFailedReduces);
pendingFailedReduces.clear(); pendingFailedReduces.clear();
} }

29
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java
View File

@ -51,11 +51,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Logger;
import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator; import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator;
import org.apache.hadoop.yarn.sls.scheduler.TaskRunner; import org.apache.hadoop.yarn.sls.scheduler.TaskRunner;
import org.apache.hadoop.yarn.sls.utils.SLSUtils; import org.apache.hadoop.yarn.sls.utils.SLSUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Private @Private
@Unstable @Unstable
@ -74,7 +74,7 @@ public class NMSimulator extends TaskRunner.Task {
private ResourceManager rm; private ResourceManager rm;
// heart beat response id // heart beat response id
private int RESPONSE_ID = 1; private int RESPONSE_ID = 1;
private final static Logger LOG = Logger.getLogger(NMSimulator.class); private final static Logger LOG = LoggerFactory.getLogger(NMSimulator.class);
public void init(String nodeIdStr, int memory, int cores, public void init(String nodeIdStr, int memory, int cores,
int dispatchTime, int heartBeatInterval, ResourceManager rm) int dispatchTime, int heartBeatInterval, ResourceManager rm)
@ -120,8 +120,7 @@ public class NMSimulator extends TaskRunner.Task {
while ((cs = containerQueue.poll()) != null) { while ((cs = containerQueue.poll()) != null) {
runningContainers.remove(cs.getId()); runningContainers.remove(cs.getId());
completedContainerList.add(cs.getId()); completedContainerList.add(cs.getId());
LOG.debug(MessageFormat.format("Container {0} has completed", LOG.debug("Container {} has completed", cs.getId());
cs.getId()));
} }
} }
@ -148,14 +147,14 @@ public class NMSimulator extends TaskRunner.Task {
synchronized(amContainerList) { synchronized(amContainerList) {
amContainerList.remove(containerId); amContainerList.remove(containerId);
} }
LOG.debug(MessageFormat.format("NodeManager {0} releases " + LOG.debug("NodeManager {} releases an AM ({}).",
"an AM ({1}).", node.getNodeID(), containerId)); node.getNodeID(), containerId);
} else { } else {
cs = runningContainers.remove(containerId); cs = runningContainers.remove(containerId);
containerQueue.remove(cs); containerQueue.remove(cs);
releasedContainerList.add(containerId); releasedContainerList.add(containerId);
LOG.debug(MessageFormat.format("NodeManager {0} releases a " + LOG.debug("NodeManager {} releases a container ({}).",
"container ({1}).", node.getNodeID(), containerId)); node.getNodeID(), containerId);
} }
} }
} }
@ -189,8 +188,8 @@ public class NMSimulator extends TaskRunner.Task {
// add complete containers // add complete containers
synchronized(completedContainerList) { synchronized(completedContainerList) {
for (ContainerId cId : completedContainerList) { for (ContainerId cId : completedContainerList) {
LOG.debug(MessageFormat.format("NodeManager {0} completed" + LOG.debug("NodeManager {} completed container ({}).",
" container ({1}).", node.getNodeID(), cId)); node.getNodeID(), cId);
csList.add(newContainerStatus( csList.add(newContainerStatus(
cId, ContainerState.COMPLETE, ContainerExitStatus.SUCCESS)); cId, ContainerState.COMPLETE, ContainerExitStatus.SUCCESS));
} }
@ -199,8 +198,8 @@ public class NMSimulator extends TaskRunner.Task {
// released containers // released containers
synchronized(releasedContainerList) { synchronized(releasedContainerList) {
for (ContainerId cId : releasedContainerList) { for (ContainerId cId : releasedContainerList) {
LOG.debug(MessageFormat.format("NodeManager {0} released container" + LOG.debug("NodeManager {} released container ({}).",
" ({1}).", node.getNodeID(), cId)); node.getNodeID(), cId);
csList.add(newContainerStatus( csList.add(newContainerStatus(
cId, ContainerState.COMPLETE, ContainerExitStatus.ABORTED)); cId, ContainerState.COMPLETE, ContainerExitStatus.ABORTED));
} }
@ -227,8 +226,8 @@ public class NMSimulator extends TaskRunner.Task {
* launch a new container with the given life time * launch a new container with the given life time
*/ */
public void addNewContainer(Container container, long lifeTimeMS) { public void addNewContainer(Container container, long lifeTimeMS) {
LOG.debug(MessageFormat.format("NodeManager {0} launches a new " + LOG.debug("NodeManager {} launches a new container ({}).",
"container ({1}).", node.getNodeID(), container.getId())); node.getNodeID(), container.getId());
if (lifeTimeMS != -1) { if (lifeTimeMS != -1) {
// normal container // normal container
ContainerSimulator cs = new ContainerSimulator(container.getId(), ContainerSimulator cs = new ContainerSimulator(container.getId(),

9
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
View File

@ -70,7 +70,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
@ -90,7 +89,6 @@ import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
import org.apache.hadoop.yarn.sls.web.SLSWebApp; import org.apache.hadoop.yarn.sls.web.SLSWebApp;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.log4j.Logger;
import com.codahale.metrics.Counter; import com.codahale.metrics.Counter;
import com.codahale.metrics.CsvReporter; import com.codahale.metrics.CsvReporter;
@ -99,6 +97,8 @@ import com.codahale.metrics.Histogram;
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.SlidingWindowReservoir; import com.codahale.metrics.SlidingWindowReservoir;
import com.codahale.metrics.Timer; import com.codahale.metrics.Timer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Private @Private
@Unstable @Unstable
@ -157,7 +157,8 @@ public class ResourceSchedulerWrapper
private Set<String> queueSet; private Set<String> queueSet;
private Set<String> trackedAppSet; private Set<String> trackedAppSet;
public final Logger LOG = Logger.getLogger(ResourceSchedulerWrapper.class); public final Logger LOG =
LoggerFactory.getLogger(ResourceSchedulerWrapper.class);
public ResourceSchedulerWrapper() { public ResourceSchedulerWrapper() {
super(ResourceSchedulerWrapper.class.getName()); super(ResourceSchedulerWrapper.class.getName());
@ -668,7 +669,7 @@ public class ResourceSchedulerWrapper
File dir = new File(metricsOutputDir + "/metrics"); File dir = new File(metricsOutputDir + "/metrics");
if(! dir.exists() if(! dir.exists()
&& ! dir.mkdirs()) { && ! dir.mkdirs()) {
LOG.error("Cannot create directory " + dir.getAbsoluteFile()); LOG.error("Cannot create directory {}", dir.getAbsoluteFile());
} }
final CsvReporter reporter = CsvReporter.forRegistry(metrics) final CsvReporter reporter = CsvReporter.forRegistry(metrics)
.formatFor(Locale.US) .formatFor(Locale.US)

7
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
View File

@ -74,7 +74,8 @@ import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
import org.apache.hadoop.yarn.sls.web.SLSWebApp; import org.apache.hadoop.yarn.sls.web.SLSWebApp;
import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.log4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SLSCapacityScheduler extends CapacityScheduler implements public class SLSCapacityScheduler extends CapacityScheduler implements
SchedulerWrapper,Configurable { SchedulerWrapper,Configurable {
@ -130,7 +131,7 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
private Set<String> queueSet; private Set<String> queueSet;
private Set<String> trackedAppSet; private Set<String> trackedAppSet;
public final Logger LOG = Logger.getLogger(SLSCapacityScheduler.class); public final Logger LOG = LoggerFactory.getLogger(SLSCapacityScheduler.class);
public SLSCapacityScheduler() { public SLSCapacityScheduler() {
samplerLock = new ReentrantLock(); samplerLock = new ReentrantLock();
@ -640,7 +641,7 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
File dir = new File(metricsOutputDir + "/metrics"); File dir = new File(metricsOutputDir + "/metrics");
if(! dir.exists() if(! dir.exists()
&& ! dir.mkdirs()) { && ! dir.mkdirs()) {
LOG.error("Cannot create directory " + dir.getAbsoluteFile()); LOG.error("Cannot create directory {}", dir.getAbsoluteFile());
} }
final CsvReporter reporter = CsvReporter.forRegistry(metrics) final CsvReporter reporter = CsvReporter.forRegistry(metrics)
.formatFor(Locale.US) .formatFor(Locale.US)