YARN-9349. Changed logging to use slf4j api.
Contributed by Prabhu Joseph
This commit is contained in:
parent
5cfb88a225
commit
2064ca015d
|
@ -167,9 +167,8 @@ public final class ProviderUtils {
|
||||||
}
|
}
|
||||||
if (clazz != null) {
|
if (clazz != null) {
|
||||||
if (fileSystemClass.isAssignableFrom(clazz)) {
|
if (fileSystemClass.isAssignableFrom(clazz)) {
|
||||||
LOG.debug("Filesystem based provider" +
|
LOG.debug("Filesystem based provider excluded from provider " +
|
||||||
" excluded from provider path due to recursive dependency: "
|
"path due to recursive dependency: {}", provider);
|
||||||
+ provider);
|
|
||||||
} else {
|
} else {
|
||||||
if (newProviderPath.length() > 0) {
|
if (newProviderPath.length() > 0) {
|
||||||
newProviderPath.append(",");
|
newProviderPath.append(",");
|
||||||
|
|
|
@ -138,20 +138,17 @@ public class ResourceUtils {
|
||||||
Map<String, ResourceInformation> res) {
|
Map<String, ResourceInformation> res) {
|
||||||
ResourceInformation ri;
|
ResourceInformation ri;
|
||||||
if (!res.containsKey(MEMORY)) {
|
if (!res.containsKey(MEMORY)) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Adding resource type - name = {}, units = {}, type = {}",
|
||||||
LOG.debug("Adding resource type - name = " + MEMORY + ", units = "
|
MEMORY, ResourceInformation.MEMORY_MB.getUnits(),
|
||||||
+ ResourceInformation.MEMORY_MB.getUnits() + ", type = "
|
ResourceTypes.COUNTABLE);
|
||||||
+ ResourceTypes.COUNTABLE);
|
|
||||||
}
|
|
||||||
ri = ResourceInformation.newInstance(MEMORY,
|
ri = ResourceInformation.newInstance(MEMORY,
|
||||||
ResourceInformation.MEMORY_MB.getUnits());
|
ResourceInformation.MEMORY_MB.getUnits());
|
||||||
res.put(MEMORY, ri);
|
res.put(MEMORY, ri);
|
||||||
}
|
}
|
||||||
if (!res.containsKey(VCORES)) {
|
if (!res.containsKey(VCORES)) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Adding resource type - name = {}, units = {}, type = {}",
|
||||||
LOG.debug("Adding resource type - name = " + VCORES
|
VCORES, ResourceInformation.VCORES.getUnits(),
|
||||||
+ ", units = , type = " + ResourceTypes.COUNTABLE);
|
ResourceTypes.COUNTABLE);
|
||||||
}
|
|
||||||
ri = ResourceInformation.newInstance(VCORES);
|
ri = ResourceInformation.newInstance(VCORES);
|
||||||
res.put(VCORES, ri);
|
res.put(VCORES, ri);
|
||||||
}
|
}
|
||||||
|
@ -189,9 +186,9 @@ public class ResourceUtils {
|
||||||
String resourceTypesKey, String schedulerKey, long schedulerDefault) {
|
String resourceTypesKey, String schedulerKey, long schedulerDefault) {
|
||||||
long value = conf.getLong(resourceTypesKey, -1L);
|
long value = conf.getLong(resourceTypesKey, -1L);
|
||||||
if (value == -1) {
|
if (value == -1) {
|
||||||
LOG.debug("Mandatory Resource '" + resourceTypesKey + "' is not "
|
LOG.debug("Mandatory Resource '{}' is not "
|
||||||
+ "configured in resource-types config file. Setting allocation "
|
+ "configured in resource-types config file. Setting allocation "
|
||||||
+ "specified using '" + schedulerKey + "'");
|
+ "specified using '{}'", resourceTypesKey, schedulerKey);
|
||||||
value = conf.getLong(schedulerKey, schedulerDefault);
|
value = conf.getLong(schedulerKey, schedulerDefault);
|
||||||
}
|
}
|
||||||
return value;
|
return value;
|
||||||
|
@ -450,9 +447,7 @@ public class ResourceUtils {
|
||||||
Configuration conf) {
|
Configuration conf) {
|
||||||
try {
|
try {
|
||||||
InputStream ris = getConfInputStream(resourceFile, conf);
|
InputStream ris = getConfInputStream(resourceFile, conf);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Found {}, adding to configuration", resourceFile);
|
||||||
LOG.debug("Found " + resourceFile + ", adding to configuration");
|
|
||||||
}
|
|
||||||
conf.addResource(ris);
|
conf.addResource(ris);
|
||||||
} catch (FileNotFoundException fe) {
|
} catch (FileNotFoundException fe) {
|
||||||
LOG.info("Unable to find '" + resourceFile + "'.");
|
LOG.info("Unable to find '" + resourceFile + "'.");
|
||||||
|
@ -575,10 +570,8 @@ public class ResourceUtils {
|
||||||
}
|
}
|
||||||
nodeResources.get(resourceType).setValue(resourceValue);
|
nodeResources.get(resourceType).setValue(resourceValue);
|
||||||
nodeResources.get(resourceType).setUnits(units);
|
nodeResources.get(resourceType).setUnits(units);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Setting value for resource type {} to {} with units {}",
|
||||||
LOG.debug("Setting value for resource type " + resourceType + " to "
|
resourceType, resourceValue, units);
|
||||||
+ resourceValue + " with units " + units);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1269,19 +1269,15 @@ public class ApplicationMaster {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void onContainerStopped(ContainerId containerId) {
|
public void onContainerStopped(ContainerId containerId) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Succeeded to stop Container {}", containerId);
|
||||||
LOG.debug("Succeeded to stop Container " + containerId);
|
|
||||||
}
|
|
||||||
containers.remove(containerId);
|
containers.remove(containerId);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void onContainerStatusReceived(ContainerId containerId,
|
public void onContainerStatusReceived(ContainerId containerId,
|
||||||
ContainerStatus containerStatus) {
|
ContainerStatus containerStatus) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Container Status: id={}, status={}", containerId,
|
||||||
LOG.debug("Container Status: id=" + containerId + ", status=" +
|
containerStatus);
|
||||||
containerStatus);
|
|
||||||
}
|
|
||||||
|
|
||||||
// If promote_opportunistic_after_start is set, automatically promote
|
// If promote_opportunistic_after_start is set, automatically promote
|
||||||
// opportunistic containers to guaranteed.
|
// opportunistic containers to guaranteed.
|
||||||
|
@ -1305,9 +1301,7 @@ public class ApplicationMaster {
|
||||||
@Override
|
@Override
|
||||||
public void onContainerStarted(ContainerId containerId,
|
public void onContainerStarted(ContainerId containerId,
|
||||||
Map<String, ByteBuffer> allServiceResponse) {
|
Map<String, ByteBuffer> allServiceResponse) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Succeeded to start Container {}", containerId);
|
||||||
LOG.debug("Succeeded to start Container " + containerId);
|
|
||||||
}
|
|
||||||
Container container = containers.get(containerId);
|
Container container = containers.get(containerId);
|
||||||
if (container != null) {
|
if (container != null) {
|
||||||
applicationMaster.nmClientAsync.getContainerStatusAsync(
|
applicationMaster.nmClientAsync.getContainerStatusAsync(
|
||||||
|
|
|
@ -361,9 +361,7 @@ public class SystemServiceManagerImpl extends AbstractService
|
||||||
private Service getServiceDefinition(Path filePath) {
|
private Service getServiceDefinition(Path filePath) {
|
||||||
Service service = null;
|
Service service = null;
|
||||||
try {
|
try {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Loading service definition from FS: {}", filePath);
|
||||||
LOG.debug("Loading service definition from FS: " + filePath);
|
|
||||||
}
|
|
||||||
service = jsonSerDeser.load(fs, filePath);
|
service = jsonSerDeser.load(fs, filePath);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
LOG.info("Error while loading service definition from FS: {}", e);
|
LOG.info("Error while loading service definition from FS: {}", e);
|
||||||
|
|
|
@ -1189,7 +1189,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
|
||||||
.append(entry.getValue().getResource().getFile())
|
.append(entry.getValue().getResource().getFile())
|
||||||
.append(System.lineSeparator());
|
.append(System.lineSeparator());
|
||||||
}
|
}
|
||||||
LOG.debug(builder.toString());
|
LOG.debug("{}", builder);
|
||||||
}
|
}
|
||||||
|
|
||||||
private String buildCommandLine(Service app, Configuration conf,
|
private String buildCommandLine(Service app, Configuration conf,
|
||||||
|
@ -1249,7 +1249,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
|
||||||
}
|
}
|
||||||
if (!UserGroupInformation.isSecurityEnabled()) {
|
if (!UserGroupInformation.isSecurityEnabled()) {
|
||||||
String userName = UserGroupInformation.getCurrentUser().getUserName();
|
String userName = UserGroupInformation.getCurrentUser().getUserName();
|
||||||
LOG.debug("Run as user " + userName);
|
LOG.debug("Run as user {}", userName);
|
||||||
// HADOOP_USER_NAME env is used by UserGroupInformation when log in
|
// HADOOP_USER_NAME env is used by UserGroupInformation when log in
|
||||||
// This env makes AM run as this user
|
// This env makes AM run as this user
|
||||||
env.put("HADOOP_USER_NAME", userName);
|
env.put("HADOOP_USER_NAME", userName);
|
||||||
|
@ -1405,7 +1405,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
if (tokens != null && tokens.length != 0) {
|
if (tokens != null && tokens.length != 0) {
|
||||||
for (Token<?> token : tokens) {
|
for (Token<?> token : tokens) {
|
||||||
LOG.debug("Got DT: " + token);
|
LOG.debug("Got DT: {}", token);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -196,7 +196,7 @@ public class AbstractLauncher {
|
||||||
|
|
||||||
String key = entry.getKey();
|
String key = entry.getKey();
|
||||||
LocalResource val = entry.getValue();
|
LocalResource val = entry.getValue();
|
||||||
log.debug(key + "=" + ServiceUtils.stringify(val.getResource()));
|
log.debug("{} = {}", key, ServiceUtils.stringify(val.getResource()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -77,16 +77,12 @@ public class ServiceMetricsSink implements MetricsSink {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isServiceMetrics && appId != null) {
|
if (isServiceMetrics && appId != null) {
|
||||||
if (log.isDebugEnabled()) {
|
log.debug("Publishing service metrics. {}", record);
|
||||||
log.debug("Publishing service metrics. " + record);
|
|
||||||
}
|
|
||||||
serviceTimelinePublisher.publishMetrics(record.metrics(), appId,
|
serviceTimelinePublisher.publishMetrics(record.metrics(), appId,
|
||||||
ServiceTimelineEntityType.SERVICE_ATTEMPT.toString(),
|
ServiceTimelineEntityType.SERVICE_ATTEMPT.toString(),
|
||||||
record.timestamp());
|
record.timestamp());
|
||||||
} else if (isComponentMetrics) {
|
} else if (isComponentMetrics) {
|
||||||
if (log.isDebugEnabled()) {
|
log.debug("Publishing Component metrics. {}", record);
|
||||||
log.debug("Publishing Component metrics. " + record);
|
|
||||||
}
|
|
||||||
serviceTimelinePublisher.publishMetrics(record.metrics(), record.name(),
|
serviceTimelinePublisher.publishMetrics(record.metrics(), record.name(),
|
||||||
ServiceTimelineEntityType.COMPONENT.toString(), record.timestamp());
|
ServiceTimelineEntityType.COMPONENT.toString(), record.timestamp());
|
||||||
}
|
}
|
||||||
|
|
|
@ -857,10 +857,7 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
|
||||||
|
|
||||||
int loggingCounter = logInterval;
|
int loggingCounter = logInterval;
|
||||||
do {
|
do {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Check the condition for main loop.");
|
||||||
LOG.debug("Check the condition for main loop.");
|
|
||||||
}
|
|
||||||
|
|
||||||
boolean result = check.get();
|
boolean result = check.get();
|
||||||
if (result) {
|
if (result) {
|
||||||
LOG.info("Exits the main loop.");
|
LOG.info("Exits the main loop.");
|
||||||
|
|
|
@ -465,10 +465,7 @@ extends AbstractService {
|
||||||
|
|
||||||
int loggingCounter = logInterval;
|
int loggingCounter = logInterval;
|
||||||
do {
|
do {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Check the condition for main loop.");
|
||||||
LOG.debug("Check the condition for main loop.");
|
|
||||||
}
|
|
||||||
|
|
||||||
boolean result = check.get();
|
boolean result = check.get();
|
||||||
if (result) {
|
if (result) {
|
||||||
LOG.info("Exits the main loop.");
|
LOG.info("Exits the main loop.");
|
||||||
|
|
|
@ -80,10 +80,8 @@ public class ContainerManagementProtocolProxy {
|
||||||
+ " (" + maxConnectedNMs + ") can not be less than 0.");
|
+ " (" + maxConnectedNMs + ") can not be less than 0.");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{} : {}", YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES,
|
||||||
LOG.debug(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES + " : " +
|
maxConnectedNMs);
|
||||||
maxConnectedNMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (maxConnectedNMs > 0) {
|
if (maxConnectedNMs > 0) {
|
||||||
cmProxy =
|
cmProxy =
|
||||||
|
@ -110,10 +108,8 @@ public class ContainerManagementProtocolProxy {
|
||||||
while (proxy != null
|
while (proxy != null
|
||||||
&& !proxy.token.getIdentifier().equals(
|
&& !proxy.token.getIdentifier().equals(
|
||||||
nmTokenCache.getToken(containerManagerBindAddr).getIdentifier())) {
|
nmTokenCache.getToken(containerManagerBindAddr).getIdentifier())) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Refreshing proxy as NMToken got updated for node : {}",
|
||||||
LOG.debug("Refreshing proxy as NMToken got updated for node : "
|
containerManagerBindAddr);
|
||||||
+ containerManagerBindAddr);
|
|
||||||
}
|
|
||||||
// Token is updated. check if anyone has already tried closing it.
|
// Token is updated. check if anyone has already tried closing it.
|
||||||
if (!proxy.scheduledForClose) {
|
if (!proxy.scheduledForClose) {
|
||||||
// try closing the proxy. Here if someone is already using it
|
// try closing the proxy. Here if someone is already using it
|
||||||
|
@ -149,10 +145,8 @@ public class ContainerManagementProtocolProxy {
|
||||||
private void addProxyToCache(String containerManagerBindAddr,
|
private void addProxyToCache(String containerManagerBindAddr,
|
||||||
ContainerManagementProtocolProxyData proxy) {
|
ContainerManagementProtocolProxyData proxy) {
|
||||||
while (cmProxy.size() >= maxConnectedNMs) {
|
while (cmProxy.size() >= maxConnectedNMs) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Cleaning up the proxy cache, size={} max={}", cmProxy.size(),
|
||||||
LOG.debug("Cleaning up the proxy cache, size=" + cmProxy.size()
|
maxConnectedNMs);
|
||||||
+ " max=" + maxConnectedNMs);
|
|
||||||
}
|
|
||||||
boolean removedProxy = false;
|
boolean removedProxy = false;
|
||||||
for (ContainerManagementProtocolProxyData otherProxy : cmProxy.values()) {
|
for (ContainerManagementProtocolProxyData otherProxy : cmProxy.values()) {
|
||||||
removedProxy = removeProxy(otherProxy);
|
removedProxy = removeProxy(otherProxy);
|
||||||
|
@ -193,9 +187,7 @@ public class ContainerManagementProtocolProxy {
|
||||||
ContainerManagementProtocolProxyData proxy) {
|
ContainerManagementProtocolProxyData proxy) {
|
||||||
proxy.activeCallers--;
|
proxy.activeCallers--;
|
||||||
if (proxy.scheduledForClose && proxy.activeCallers < 0) {
|
if (proxy.scheduledForClose && proxy.activeCallers < 0) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Closing proxy : {}", proxy.containerManagerBindAddr);
|
||||||
LOG.debug("Closing proxy : " + proxy.containerManagerBindAddr);
|
|
||||||
}
|
|
||||||
cmProxy.remove(proxy.containerManagerBindAddr);
|
cmProxy.remove(proxy.containerManagerBindAddr);
|
||||||
try {
|
try {
|
||||||
rpc.stopProxy(proxy.getContainerManagementProtocol(), conf);
|
rpc.stopProxy(proxy.getContainerManagementProtocol(), conf);
|
||||||
|
@ -265,9 +257,7 @@ public class ContainerManagementProtocolProxy {
|
||||||
|
|
||||||
final InetSocketAddress cmAddr =
|
final InetSocketAddress cmAddr =
|
||||||
NetUtils.createSocketAddr(containerManagerBindAddr);
|
NetUtils.createSocketAddr(containerManagerBindAddr);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Opening proxy : {}", containerManagerBindAddr);
|
||||||
LOG.debug("Opening proxy : " + containerManagerBindAddr);
|
|
||||||
}
|
|
||||||
// the user in createRemoteUser in this context has to be ContainerID
|
// the user in createRemoteUser in this context has to be ContainerID
|
||||||
UserGroupInformation user =
|
UserGroupInformation user =
|
||||||
UserGroupInformation.createRemoteUser(containerId
|
UserGroupInformation.createRemoteUser(containerId
|
||||||
|
|
|
@ -137,27 +137,21 @@ class RemoteRequestsTable<T> implements Iterable<ResourceRequestInfo>{
|
||||||
if (locationMap == null) {
|
if (locationMap == null) {
|
||||||
locationMap = new HashMap<>();
|
locationMap = new HashMap<>();
|
||||||
this.remoteRequestsTable.put(priority, locationMap);
|
this.remoteRequestsTable.put(priority, locationMap);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Added priority={}", priority);
|
||||||
LOG.debug("Added priority=" + priority);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Map<ExecutionType, TreeMap<Resource, ResourceRequestInfo>>
|
Map<ExecutionType, TreeMap<Resource, ResourceRequestInfo>>
|
||||||
execTypeMap = locationMap.get(resourceName);
|
execTypeMap = locationMap.get(resourceName);
|
||||||
if (execTypeMap == null) {
|
if (execTypeMap == null) {
|
||||||
execTypeMap = new HashMap<>();
|
execTypeMap = new HashMap<>();
|
||||||
locationMap.put(resourceName, execTypeMap);
|
locationMap.put(resourceName, execTypeMap);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Added resourceName={}", resourceName);
|
||||||
LOG.debug("Added resourceName=" + resourceName);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
TreeMap<Resource, ResourceRequestInfo> capabilityMap =
|
TreeMap<Resource, ResourceRequestInfo> capabilityMap =
|
||||||
execTypeMap.get(execType);
|
execTypeMap.get(execType);
|
||||||
if (capabilityMap == null) {
|
if (capabilityMap == null) {
|
||||||
capabilityMap = new TreeMap<>(new AMRMClientImpl.ResourceReverseComparator());
|
capabilityMap = new TreeMap<>(new AMRMClientImpl.ResourceReverseComparator());
|
||||||
execTypeMap.put(execType, capabilityMap);
|
execTypeMap.put(execType, capabilityMap);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Added Execution Type={}", execType);
|
||||||
LOG.debug("Added Execution Type=" + execType);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
capabilityMap.put(capability, resReqInfo);
|
capabilityMap.put(capability, resReqInfo);
|
||||||
}
|
}
|
||||||
|
@ -168,25 +162,19 @@ class RemoteRequestsTable<T> implements Iterable<ResourceRequestInfo>{
|
||||||
Map<String, Map<ExecutionType, TreeMap<Resource,
|
Map<String, Map<ExecutionType, TreeMap<Resource,
|
||||||
ResourceRequestInfo>>> locationMap = remoteRequestsTable.get(priority);
|
ResourceRequestInfo>>> locationMap = remoteRequestsTable.get(priority);
|
||||||
if (locationMap == null) {
|
if (locationMap == null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("No such priority={}", priority);
|
||||||
LOG.debug("No such priority=" + priority);
|
|
||||||
}
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
Map<ExecutionType, TreeMap<Resource, ResourceRequestInfo>>
|
Map<ExecutionType, TreeMap<Resource, ResourceRequestInfo>>
|
||||||
execTypeMap = locationMap.get(resourceName);
|
execTypeMap = locationMap.get(resourceName);
|
||||||
if (execTypeMap == null) {
|
if (execTypeMap == null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("No such resourceName={}", resourceName);
|
||||||
LOG.debug("No such resourceName=" + resourceName);
|
|
||||||
}
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
TreeMap<Resource, ResourceRequestInfo> capabilityMap =
|
TreeMap<Resource, ResourceRequestInfo> capabilityMap =
|
||||||
execTypeMap.get(execType);
|
execTypeMap.get(execType);
|
||||||
if (capabilityMap == null) {
|
if (capabilityMap == null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("No such Execution Type={}", execType);
|
||||||
LOG.debug("No such Execution Type=" + execType);
|
|
||||||
}
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
retVal = capabilityMap.remove(capability);
|
retVal = capabilityMap.remove(capability);
|
||||||
|
@ -286,9 +274,8 @@ class RemoteRequestsTable<T> implements Iterable<ResourceRequestInfo>{
|
||||||
if (ResourceRequest.ANY.equals(resourceName)) {
|
if (ResourceRequest.ANY.equals(resourceName)) {
|
||||||
resourceRequestInfo.remoteRequest.setNodeLabelExpression(labelExpression);
|
resourceRequestInfo.remoteRequest.setNodeLabelExpression(labelExpression);
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Adding request to ask {}", resourceRequestInfo.remoteRequest);
|
||||||
LOG.debug("Adding request to ask " + resourceRequestInfo.remoteRequest);
|
|
||||||
}
|
|
||||||
return resourceRequestInfo;
|
return resourceRequestInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -298,22 +285,16 @@ class RemoteRequestsTable<T> implements Iterable<ResourceRequestInfo>{
|
||||||
execTypeReq.getExecutionType(), capability);
|
execTypeReq.getExecutionType(), capability);
|
||||||
|
|
||||||
if (resourceRequestInfo == null) {
|
if (resourceRequestInfo == null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Not decrementing resource as ResourceRequestInfo with"
|
||||||
LOG.debug("Not decrementing resource as ResourceRequestInfo with" +
|
+ " priority={} resourceName={} executionType={} capability={} is"
|
||||||
"priority=" + priority + ", " +
|
+ " not present in request table", priority, resourceName,
|
||||||
"resourceName=" + resourceName + ", " +
|
execTypeReq, capability);
|
||||||
"executionType=" + execTypeReq + ", " +
|
|
||||||
"capability=" + capability + " is not present in request table");
|
|
||||||
}
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("BEFORE decResourceRequest: applicationId= priority={}"
|
||||||
LOG.debug("BEFORE decResourceRequest:" + " applicationId="
|
+" resourceName={} numContainers={}", priority.getPriority(),
|
||||||
+ " priority=" + priority.getPriority()
|
resourceName, resourceRequestInfo.remoteRequest.getNumContainers());
|
||||||
+ " resourceName=" + resourceName + " numContainers="
|
|
||||||
+ resourceRequestInfo.remoteRequest.getNumContainers());
|
|
||||||
}
|
|
||||||
|
|
||||||
resourceRequestInfo.remoteRequest.setNumContainers(
|
resourceRequestInfo.remoteRequest.setNumContainers(
|
||||||
resourceRequestInfo.remoteRequest.getNumContainers() - 1);
|
resourceRequestInfo.remoteRequest.getNumContainers() - 1);
|
||||||
|
|
|
@ -84,9 +84,7 @@ public class SharedCacheClientImpl extends SharedCacheClient {
|
||||||
@Override
|
@Override
|
||||||
protected void serviceStart() throws Exception {
|
protected void serviceStart() throws Exception {
|
||||||
this.scmClient = createClientProxy();
|
this.scmClient = createClientProxy();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Connecting to Shared Cache Manager at {}", this.scmAddress);
|
||||||
LOG.debug("Connecting to Shared Cache Manager at " + this.scmAddress);
|
|
||||||
}
|
|
||||||
super.serviceStart();
|
super.serviceStart();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -397,10 +397,8 @@ public class YarnClientImpl extends YarnClient {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
credentials.addToken(timelineService, timelineDelegationToken);
|
credentials.addToken(timelineService, timelineDelegationToken);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Add timeline delegation token into credentials: {}",
|
||||||
LOG.debug("Add timeline delegation token into credentials: "
|
timelineDelegationToken);
|
||||||
+ timelineDelegationToken);
|
|
||||||
}
|
|
||||||
DataOutputBuffer dob = new DataOutputBuffer();
|
DataOutputBuffer dob = new DataOutputBuffer();
|
||||||
credentials.writeTokenStorageToStream(dob);
|
credentials.writeTokenStorageToStream(dob);
|
||||||
tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
|
tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
|
||||||
|
|
|
@ -224,10 +224,8 @@ public class FileSystemTimelineWriter extends TimelineWriter{
|
||||||
if (!entitiesToSummaryCache.isEmpty()) {
|
if (!entitiesToSummaryCache.isEmpty()) {
|
||||||
Path summaryLogPath =
|
Path summaryLogPath =
|
||||||
new Path(attemptDir, SUMMARY_LOG_PREFIX + appAttemptId.toString());
|
new Path(attemptDir, SUMMARY_LOG_PREFIX + appAttemptId.toString());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Writing summary log for {} to {}", appAttemptId,
|
||||||
LOG.debug("Writing summary log for " + appAttemptId.toString() + " to "
|
summaryLogPath);
|
||||||
+ summaryLogPath);
|
|
||||||
}
|
|
||||||
this.logFDsCache.writeSummaryEntityLogs(fs, summaryLogPath, objMapper,
|
this.logFDsCache.writeSummaryEntityLogs(fs, summaryLogPath, objMapper,
|
||||||
appAttemptId, entitiesToSummaryCache, isAppendSupported);
|
appAttemptId, entitiesToSummaryCache, isAppendSupported);
|
||||||
}
|
}
|
||||||
|
@ -235,10 +233,7 @@ public class FileSystemTimelineWriter extends TimelineWriter{
|
||||||
if (!entitiesToEntityCache.isEmpty()) {
|
if (!entitiesToEntityCache.isEmpty()) {
|
||||||
Path entityLogPath =
|
Path entityLogPath =
|
||||||
new Path(attemptDir, ENTITY_LOG_PREFIX + groupId.toString());
|
new Path(attemptDir, ENTITY_LOG_PREFIX + groupId.toString());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Writing entity log for {} to {}", groupId, entityLogPath);
|
||||||
LOG.debug("Writing entity log for " + groupId.toString() + " to "
|
|
||||||
+ entityLogPath);
|
|
||||||
}
|
|
||||||
this.logFDsCache.writeEntityLogs(fs, entityLogPath, objMapper,
|
this.logFDsCache.writeEntityLogs(fs, entityLogPath, objMapper,
|
||||||
appAttemptId, groupId, entitiesToEntityCache, isAppendSupported);
|
appAttemptId, groupId, entitiesToEntityCache, isAppendSupported);
|
||||||
}
|
}
|
||||||
|
@ -293,8 +288,7 @@ public class FileSystemTimelineWriter extends TimelineWriter{
|
||||||
new Path(attemptDirCache.getAppAttemptDir(appAttemptId),
|
new Path(attemptDirCache.getAppAttemptDir(appAttemptId),
|
||||||
DOMAIN_LOG_PREFIX + appAttemptId.toString());
|
DOMAIN_LOG_PREFIX + appAttemptId.toString());
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug("Writing domains for " + appAttemptId.toString() + " to "
|
LOG.debug("Writing domains for {} to {}", appAttemptId, domainLogPath);
|
||||||
+ domainLogPath);
|
|
||||||
}
|
}
|
||||||
this.logFDsCache.writeDomainLog(
|
this.logFDsCache.writeDomainLog(
|
||||||
fs, domainLogPath, objMapper, domain, isAppendSupported);
|
fs, domainLogPath, objMapper, domain, isAppendSupported);
|
||||||
|
@ -324,9 +318,7 @@ public class FileSystemTimelineWriter extends TimelineWriter{
|
||||||
if (writerClosed()) {
|
if (writerClosed()) {
|
||||||
prepareForWrite();
|
prepareForWrite();
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Writing entity list of size {}", entities.size());
|
||||||
LOG.debug("Writing entity list of size " + entities.size());
|
|
||||||
}
|
|
||||||
for (TimelineEntity entity : entities) {
|
for (TimelineEntity entity : entities) {
|
||||||
getObjectMapper().writeValue(getJsonGenerator(), entity);
|
getObjectMapper().writeValue(getJsonGenerator(), entity);
|
||||||
}
|
}
|
||||||
|
@ -558,9 +550,7 @@ public class FileSystemTimelineWriter extends TimelineWriter{
|
||||||
try {
|
try {
|
||||||
flush();
|
flush();
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{}", e);
|
||||||
LOG.debug(e.toString());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -997,9 +987,7 @@ public class FileSystemTimelineWriter extends TimelineWriter{
|
||||||
Path attemptDir = new Path(appDir, appAttemptId.toString());
|
Path attemptDir = new Path(appDir, appAttemptId.toString());
|
||||||
if (FileSystem.mkdirs(fs, attemptDir,
|
if (FileSystem.mkdirs(fs, attemptDir,
|
||||||
new FsPermission(APP_LOG_DIR_PERMISSIONS))) {
|
new FsPermission(APP_LOG_DIR_PERMISSIONS))) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("New attempt directory created - {}", attemptDir);
|
||||||
LOG.debug("New attempt directory created - " + attemptDir);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return attemptDir;
|
return attemptDir;
|
||||||
}
|
}
|
||||||
|
@ -1009,9 +997,7 @@ public class FileSystemTimelineWriter extends TimelineWriter{
|
||||||
Path appDir = new Path(appRootDir, appId.toString());
|
Path appDir = new Path(appRootDir, appId.toString());
|
||||||
if (FileSystem.mkdirs(fs, appDir,
|
if (FileSystem.mkdirs(fs, appDir,
|
||||||
new FsPermission(APP_LOG_DIR_PERMISSIONS))) {
|
new FsPermission(APP_LOG_DIR_PERMISSIONS))) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("New app directory created - {}", appDir);
|
||||||
LOG.debug("New app directory created - " + appDir);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return appDir;
|
return appDir;
|
||||||
}
|
}
|
||||||
|
@ -1023,9 +1009,7 @@ public class FileSystemTimelineWriter extends TimelineWriter{
|
||||||
Path userDir = new Path(activePath, user);
|
Path userDir = new Path(activePath, user);
|
||||||
if (FileSystem.mkdirs(fs, userDir,
|
if (FileSystem.mkdirs(fs, userDir,
|
||||||
new FsPermission(APP_LOG_DIR_PERMISSIONS))) {
|
new FsPermission(APP_LOG_DIR_PERMISSIONS))) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("New user directory created - {}", userDir);
|
||||||
LOG.debug("New user directory created - " + userDir);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return userDir;
|
return userDir;
|
||||||
}
|
}
|
||||||
|
|
|
@ -133,11 +133,8 @@ public abstract class TimelineWriter implements Flushable {
|
||||||
LOG.error(msg);
|
LOG.error(msg);
|
||||||
if (resp != null) {
|
if (resp != null) {
|
||||||
msg += " HTTP error code: " + resp.getStatus();
|
msg += " HTTP error code: " + resp.getStatus();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("HTTP error code: {} Server response : \n{}",
|
||||||
String output = resp.getEntity(String.class);
|
resp.getStatus(), resp.getEntity(String.class));
|
||||||
LOG.debug("HTTP error code: " + resp.getStatus()
|
|
||||||
+ " Server response : \n" + output);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
throw new YarnException(msg);
|
throw new YarnException(msg);
|
||||||
}
|
}
|
||||||
|
@ -149,18 +146,14 @@ public abstract class TimelineWriter implements Flushable {
|
||||||
public ClientResponse doPostingObject(Object object, String path) {
|
public ClientResponse doPostingObject(Object object, String path) {
|
||||||
WebResource webResource = client.resource(resURI);
|
WebResource webResource = client.resource(resURI);
|
||||||
if (path == null) {
|
if (path == null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("POST to {}", resURI);
|
||||||
LOG.debug("POST to " + resURI);
|
|
||||||
}
|
|
||||||
ClientResponse r = webResource.accept(MediaType.APPLICATION_JSON)
|
ClientResponse r = webResource.accept(MediaType.APPLICATION_JSON)
|
||||||
.type(MediaType.APPLICATION_JSON)
|
.type(MediaType.APPLICATION_JSON)
|
||||||
.post(ClientResponse.class, object);
|
.post(ClientResponse.class, object);
|
||||||
r.bufferEntity();
|
r.bufferEntity();
|
||||||
return r;
|
return r;
|
||||||
} else if (path.equals("domain")) {
|
} else if (path.equals("domain")) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("PUT to {}/{}", resURI, path);
|
||||||
LOG.debug("PUT to " + resURI +"/" + path);
|
|
||||||
}
|
|
||||||
ClientResponse r = webResource.path(path).accept(MediaType.APPLICATION_JSON)
|
ClientResponse r = webResource.path(path).accept(MediaType.APPLICATION_JSON)
|
||||||
.type(MediaType.APPLICATION_JSON)
|
.type(MediaType.APPLICATION_JSON)
|
||||||
.put(ClientResponse.class, object);
|
.put(ClientResponse.class, object);
|
||||||
|
|
|
@ -189,10 +189,8 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
protected void dispatch(Event event) {
|
protected void dispatch(Event event) {
|
||||||
//all events go thru this loop
|
//all events go thru this loop
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Dispatching the event {}.{}", event.getClass().getName(),
|
||||||
LOG.debug("Dispatching the event " + event.getClass().getName() + "."
|
event);
|
||||||
+ event.toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
Class<? extends Enum> type = event.getType().getDeclaringClass();
|
Class<? extends Enum> type = event.getType().getDeclaringClass();
|
||||||
|
|
||||||
|
|
|
@ -45,7 +45,7 @@ public class HadoopYarnProtoRPC extends YarnRPC {
|
||||||
@Override
|
@Override
|
||||||
public Object getProxy(Class protocol, InetSocketAddress addr,
|
public Object getProxy(Class protocol, InetSocketAddress addr,
|
||||||
Configuration conf) {
|
Configuration conf) {
|
||||||
LOG.debug("Creating a HadoopYarnProtoRpc proxy for protocol " + protocol);
|
LOG.debug("Creating a HadoopYarnProtoRpc proxy for protocol {}", protocol);
|
||||||
return RpcFactoryProvider.getClientFactory(conf).getClient(protocol, 1,
|
return RpcFactoryProvider.getClientFactory(conf).getClient(protocol, 1,
|
||||||
addr, conf);
|
addr, conf);
|
||||||
}
|
}
|
||||||
|
@ -60,8 +60,8 @@ public class HadoopYarnProtoRPC extends YarnRPC {
|
||||||
InetSocketAddress addr, Configuration conf,
|
InetSocketAddress addr, Configuration conf,
|
||||||
SecretManager<? extends TokenIdentifier> secretManager,
|
SecretManager<? extends TokenIdentifier> secretManager,
|
||||||
int numHandlers, String portRangeConfig) {
|
int numHandlers, String portRangeConfig) {
|
||||||
LOG.debug("Creating a HadoopYarnProtoRpc server for protocol " + protocol +
|
LOG.debug("Creating a HadoopYarnProtoRpc server for protocol {} with {}"
|
||||||
" with " + numHandlers + " handlers");
|
+ " handlers", protocol, numHandlers);
|
||||||
|
|
||||||
return RpcFactoryProvider.getServerFactory(conf).getServer(protocol,
|
return RpcFactoryProvider.getServerFactory(conf).getServer(protocol,
|
||||||
instance, addr, conf, secretManager, numHandlers, portRangeConfig);
|
instance, addr, conf, secretManager, numHandlers, portRangeConfig);
|
||||||
|
|
|
@ -57,7 +57,7 @@ public abstract class YarnRPC {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static YarnRPC create(Configuration conf) {
|
public static YarnRPC create(Configuration conf) {
|
||||||
LOG.debug("Creating YarnRPC for " +
|
LOG.debug("Creating YarnRPC for {}",
|
||||||
conf.get(YarnConfiguration.IPC_RPC_IMPL));
|
conf.get(YarnConfiguration.IPC_RPC_IMPL));
|
||||||
String clazzName = conf.get(YarnConfiguration.IPC_RPC_IMPL);
|
String clazzName = conf.get(YarnConfiguration.IPC_RPC_IMPL);
|
||||||
if (clazzName == null) {
|
if (clazzName == null) {
|
||||||
|
|
|
@ -850,10 +850,10 @@ public class LogAggregationIndexedFileController
|
||||||
}
|
}
|
||||||
if (uuidReadLen != UUID_LENGTH || !Arrays.equals(this.uuid, uuidRead)) {
|
if (uuidReadLen != UUID_LENGTH || !Arrays.equals(this.uuid, uuidRead)) {
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug("the length of loaded UUID:" + uuidReadLen);
|
LOG.debug("the length of loaded UUID:{}", uuidReadLen);
|
||||||
LOG.debug("the loaded UUID:" + new String(uuidRead,
|
LOG.debug("the loaded UUID:{}", new String(uuidRead,
|
||||||
Charset.forName("UTF-8")));
|
Charset.forName("UTF-8")));
|
||||||
LOG.debug("the expected UUID:" + new String(this.uuid,
|
LOG.debug("the expected UUID:{}", new String(this.uuid,
|
||||||
Charset.forName("UTF-8")));
|
Charset.forName("UTF-8")));
|
||||||
}
|
}
|
||||||
throw new IOException("The UUID from "
|
throw new IOException("The UUID from "
|
||||||
|
|
|
@ -61,9 +61,7 @@ public class NonAppendableFSNodeLabelStore extends FileSystemNodeLabelsStore {
|
||||||
fs.delete(oldMirrorPath, false);
|
fs.delete(oldMirrorPath, false);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
// do nothing
|
// do nothing
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Exception while removing old mirror", e);
|
||||||
LOG.debug("Exception while removing old mirror", e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// rename new to old
|
// rename new to old
|
||||||
|
|
|
@ -43,10 +43,10 @@ public class AMRMTokenSelector implements
|
||||||
if (service == null) {
|
if (service == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
LOG.debug("Looking for a token with service " + service.toString());
|
LOG.debug("Looking for a token with service {}", service);
|
||||||
for (Token<? extends TokenIdentifier> token : tokens) {
|
for (Token<? extends TokenIdentifier> token : tokens) {
|
||||||
LOG.debug("Token kind is " + token.getKind().toString()
|
LOG.debug("Token kind is {} and the token's service name is {}",
|
||||||
+ " and the token's service name is " + token.getService());
|
token.getKind(), token.getService());
|
||||||
if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())
|
if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())
|
||||||
&& checkService(service, token)) {
|
&& checkService(service, token)) {
|
||||||
return (Token<AMRMTokenIdentifier>) token;
|
return (Token<AMRMTokenIdentifier>) token;
|
||||||
|
|
|
@ -326,7 +326,7 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void write(DataOutput out) throws IOException {
|
public void write(DataOutput out) throws IOException {
|
||||||
LOG.debug("Writing ContainerTokenIdentifier to RPC layer: " + this);
|
LOG.debug("Writing ContainerTokenIdentifier to RPC layer: {}", this);
|
||||||
out.write(proto.toByteArray());
|
out.write(proto.toByteArray());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -45,10 +45,8 @@ public class ContainerTokenSelector implements
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
for (Token<? extends TokenIdentifier> token : tokens) {
|
for (Token<? extends TokenIdentifier> token : tokens) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Looking for service: {}. Current token is {}", service,
|
||||||
LOG.debug("Looking for service: " + service + ". Current token is "
|
token);
|
||||||
+ token);
|
|
||||||
}
|
|
||||||
if (ContainerTokenIdentifier.KIND.equals(token.getKind()) &&
|
if (ContainerTokenIdentifier.KIND.equals(token.getKind()) &&
|
||||||
service.equals(token.getService())) {
|
service.equals(token.getService())) {
|
||||||
return (Token<ContainerTokenIdentifier>) token;
|
return (Token<ContainerTokenIdentifier>) token;
|
||||||
|
|
|
@ -98,7 +98,7 @@ public class NMTokenIdentifier extends TokenIdentifier {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void write(DataOutput out) throws IOException {
|
public void write(DataOutput out) throws IOException {
|
||||||
LOG.debug("Writing NMTokenIdentifier to RPC layer: " + this);
|
LOG.debug("Writing NMTokenIdentifier to RPC layer: {}", this);
|
||||||
out.write(proto.toByteArray());
|
out.write(proto.toByteArray());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -41,10 +41,8 @@ public class NMTokenSelector implements
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
for (Token<? extends TokenIdentifier> token : tokens) {
|
for (Token<? extends TokenIdentifier> token : tokens) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Looking for service: {}. Current token is {}", service,
|
||||||
LOG.debug("Looking for service: " + service + ". Current token is "
|
token);
|
||||||
+ token);
|
|
||||||
}
|
|
||||||
if (NMTokenIdentifier.KIND.equals(token.getKind()) &&
|
if (NMTokenIdentifier.KIND.equals(token.getKind()) &&
|
||||||
service.equals(token.getService())) {
|
service.equals(token.getService())) {
|
||||||
return (Token<NMTokenIdentifier>) token;
|
return (Token<NMTokenIdentifier>) token;
|
||||||
|
|
|
@ -70,7 +70,7 @@ public abstract class YarnAuthorizationProvider {
|
||||||
public static void destroy() {
|
public static void destroy() {
|
||||||
synchronized (YarnAuthorizationProvider.class) {
|
synchronized (YarnAuthorizationProvider.class) {
|
||||||
if (authorizer != null) {
|
if (authorizer != null) {
|
||||||
LOG.debug(authorizer.getClass().getName() + " is destroyed.");
|
LOG.debug("{} is destroyed.", authorizer.getClass().getName());
|
||||||
authorizer = null;
|
authorizer = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -39,10 +39,10 @@ public class ClientToAMTokenSelector implements
|
||||||
if (service == null) {
|
if (service == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
LOG.debug("Looking for a token with service " + service.toString());
|
LOG.debug("Looking for a token with service {}", service);
|
||||||
for (Token<? extends TokenIdentifier> token : tokens) {
|
for (Token<? extends TokenIdentifier> token : tokens) {
|
||||||
LOG.debug("Token kind is " + token.getKind().toString()
|
LOG.debug("Token kind is {} and the token's service name is {}",
|
||||||
+ " and the token's service name is " + token.getService());
|
token.getKind(), token.getService());
|
||||||
if (ClientToAMTokenIdentifier.KIND_NAME.equals(token.getKind())
|
if (ClientToAMTokenIdentifier.KIND_NAME.equals(token.getKind())
|
||||||
&& service.equals(token.getService())) {
|
&& service.equals(token.getService())) {
|
||||||
return (Token<ClientToAMTokenIdentifier>) token;
|
return (Token<ClientToAMTokenIdentifier>) token;
|
||||||
|
|
|
@ -51,10 +51,10 @@ public class RMDelegationTokenSelector implements
|
||||||
if (service == null) {
|
if (service == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
LOG.debug("Looking for a token with service " + service.toString());
|
LOG.debug("Looking for a token with service {}", service);
|
||||||
for (Token<? extends TokenIdentifier> token : tokens) {
|
for (Token<? extends TokenIdentifier> token : tokens) {
|
||||||
LOG.debug("Token kind is " + token.getKind().toString()
|
LOG.debug("Token kind is {} and the token's service name is {}",
|
||||||
+ " and the token's service name is " + token.getService());
|
token.getKind(), token.getService());
|
||||||
if (RMDelegationTokenIdentifier.KIND_NAME.equals(token.getKind())
|
if (RMDelegationTokenIdentifier.KIND_NAME.equals(token.getKind())
|
||||||
&& checkService(service, token)) {
|
&& checkService(service, token)) {
|
||||||
return (Token<RMDelegationTokenIdentifier>) token;
|
return (Token<RMDelegationTokenIdentifier>) token;
|
||||||
|
|
|
@ -43,14 +43,10 @@ public class TimelineDelegationTokenSelector
|
||||||
if (service == null) {
|
if (service == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Looking for a token with service {}", service);
|
||||||
LOG.debug("Looking for a token with service " + service.toString());
|
|
||||||
}
|
|
||||||
for (Token<? extends TokenIdentifier> token : tokens) {
|
for (Token<? extends TokenIdentifier> token : tokens) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Token kind is {} and the token's service name is {}",
|
||||||
LOG.debug("Token kind is " + token.getKind().toString()
|
token.getKind(), token.getService());
|
||||||
+ " and the token's service name is " + token.getService());
|
|
||||||
}
|
|
||||||
if (TimelineDelegationTokenIdentifier.KIND_NAME.equals(token.getKind())
|
if (TimelineDelegationTokenIdentifier.KIND_NAME.equals(token.getKind())
|
||||||
&& service.equals(token.getService())) {
|
&& service.equals(token.getService())) {
|
||||||
return (Token<TimelineDelegationTokenIdentifier>) token;
|
return (Token<TimelineDelegationTokenIdentifier>) token;
|
||||||
|
|
|
@ -98,11 +98,8 @@ public class ApplicationACLsManager {
|
||||||
ApplicationAccessType applicationAccessType, String applicationOwner,
|
ApplicationAccessType applicationAccessType, String applicationOwner,
|
||||||
ApplicationId applicationId) {
|
ApplicationId applicationId) {
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Verifying access-type {} for {} on application {} owned by {}",
|
||||||
LOG.debug("Verifying access-type " + applicationAccessType + " for "
|
applicationAccessType, callerUGI, applicationId, applicationOwner);
|
||||||
+ callerUGI + " on application " + applicationId + " owned by "
|
|
||||||
+ applicationOwner);
|
|
||||||
}
|
|
||||||
|
|
||||||
String user = callerUGI.getShortUserName();
|
String user = callerUGI.getShortUserName();
|
||||||
if (!areACLsEnabled()) {
|
if (!areACLsEnabled()) {
|
||||||
|
@ -112,21 +109,18 @@ public class ApplicationACLsManager {
|
||||||
Map<ApplicationAccessType, AccessControlList> acls = this.applicationACLS
|
Map<ApplicationAccessType, AccessControlList> acls = this.applicationACLS
|
||||||
.get(applicationId);
|
.get(applicationId);
|
||||||
if (acls == null) {
|
if (acls == null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("ACL not found for application {} owned by {}."
|
||||||
LOG.debug("ACL not found for application "
|
+ " Using default [{}]", applicationId, applicationOwner,
|
||||||
+ applicationId + " owned by "
|
YarnConfiguration.DEFAULT_YARN_APP_ACL);
|
||||||
+ applicationOwner + ". Using default ["
|
|
||||||
+ YarnConfiguration.DEFAULT_YARN_APP_ACL + "]");
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
AccessControlList applicationACLInMap = acls.get(applicationAccessType);
|
AccessControlList applicationACLInMap = acls.get(applicationAccessType);
|
||||||
if (applicationACLInMap != null) {
|
if (applicationACLInMap != null) {
|
||||||
applicationACL = applicationACLInMap;
|
applicationACL = applicationACLInMap;
|
||||||
} else if (LOG.isDebugEnabled()) {
|
} else {
|
||||||
LOG.debug("ACL not found for access-type " + applicationAccessType
|
LOG.debug("ACL not found for access-type {} for application {}"
|
||||||
+ " for application " + applicationId + " owned by "
|
+ " owned by {}. Using default [{}]", applicationAccessType,
|
||||||
+ applicationOwner + ". Using default ["
|
applicationId, applicationOwner,
|
||||||
+ YarnConfiguration.DEFAULT_YARN_APP_ACL + "]");
|
YarnConfiguration.DEFAULT_YARN_APP_ACL);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -141,7 +141,7 @@ public final class DockerClientConfigHandler {
|
||||||
tokens.rewind();
|
tokens.rewind();
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
for (Token token : credentials.getAllTokens()) {
|
for (Token token : credentials.getAllTokens()) {
|
||||||
LOG.debug("Token read from token storage: " + token.toString());
|
LOG.debug("Token read from token storage: {}", token);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return credentials;
|
return credentials;
|
||||||
|
@ -172,9 +172,7 @@ public final class DockerClientConfigHandler {
|
||||||
registryUrlNode.put(ti.getRegistryUrl(), registryCredNode);
|
registryUrlNode.put(ti.getRegistryUrl(), registryCredNode);
|
||||||
registryCredNode.put(CONFIG_AUTH_KEY,
|
registryCredNode.put(CONFIG_AUTH_KEY,
|
||||||
new String(tk.getPassword(), Charset.forName("UTF-8")));
|
new String(tk.getPassword(), Charset.forName("UTF-8")));
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Prepared token for write: {}", tk);
|
||||||
LOG.debug("Prepared token for write: " + tk.toString());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (foundDockerCred) {
|
if (foundDockerCred) {
|
||||||
|
|
|
@ -394,12 +394,8 @@ public class FSDownload implements Callable<Path> {
|
||||||
throw new IOException("Invalid resource", e);
|
throw new IOException("Invalid resource", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Starting to download {} {} {}", sCopy,
|
||||||
LOG.debug(String.format("Starting to download %s %s %s",
|
resource.getType(), resource.getPattern());
|
||||||
sCopy,
|
|
||||||
resource.getType(),
|
|
||||||
resource.getPattern()));
|
|
||||||
}
|
|
||||||
|
|
||||||
final Path destinationTmp = new Path(destDirPath + "_tmp");
|
final Path destinationTmp = new Path(destDirPath + "_tmp");
|
||||||
createDir(destinationTmp, cachePerms);
|
createDir(destinationTmp, cachePerms);
|
||||||
|
@ -420,10 +416,8 @@ public class FSDownload implements Callable<Path> {
|
||||||
changePermissions(dFinal.getFileSystem(conf), dFinal);
|
changePermissions(dFinal.getFileSystem(conf), dFinal);
|
||||||
files.rename(destinationTmp, destDirPath, Rename.OVERWRITE);
|
files.rename(destinationTmp, destDirPath, Rename.OVERWRITE);
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("File has been downloaded to {} from {}",
|
||||||
LOG.debug(String.format("File has been downloaded to %s from %s",
|
new Path(destDirPath, sCopy.getName()), sCopy);
|
||||||
new Path(destDirPath, sCopy.getName()), sCopy));
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
try {
|
try {
|
||||||
files.delete(destDirPath, true);
|
files.delete(destDirPath, true);
|
||||||
|
@ -470,9 +464,7 @@ public class FSDownload implements Callable<Path> {
|
||||||
perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
|
perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Changing permissions for path {} to perm {}", path, perm);
|
||||||
LOG.debug("Changing permissions for path " + path + " to perm " + perm);
|
|
||||||
}
|
|
||||||
|
|
||||||
final FsPermission fPerm = perm;
|
final FsPermission fPerm = perm;
|
||||||
if (null == userUgi) {
|
if (null == userUgi) {
|
||||||
|
|
|
@ -264,7 +264,7 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
LOG.debug(this.toString());
|
LOG.debug("{}", this);
|
||||||
|
|
||||||
if (smapsEnabled) {
|
if (smapsEnabled) {
|
||||||
// Update smaps info
|
// Update smaps info
|
||||||
|
@ -403,13 +403,10 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
|
||||||
// memory reclaimable by killing the process
|
// memory reclaimable by killing the process
|
||||||
total += info.anonymous;
|
total += info.anonymous;
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug(" total({}): PID : {}, info : {}, total : {}",
|
||||||
LOG.debug(" total(" + olderThanAge + "): PID : " + p.getPid()
|
olderThanAge, p.getPid(), info, (total * KB_TO_BYTES));
|
||||||
+ ", info : " + info.toString()
|
|
||||||
+ ", total : " + (total * KB_TO_BYTES));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
LOG.debug(procMemInfo.toString());
|
LOG.debug("{}", procMemInfo);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -468,9 +465,7 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
|
||||||
@Override
|
@Override
|
||||||
public float getCpuUsagePercent() {
|
public float getCpuUsagePercent() {
|
||||||
BigInteger processTotalJiffies = getTotalProcessJiffies();
|
BigInteger processTotalJiffies = getTotalProcessJiffies();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Process {} jiffies:{}", pid, processTotalJiffies);
|
||||||
LOG.debug("Process " + pid + " jiffies:" + processTotalJiffies);
|
|
||||||
}
|
|
||||||
cpuTimeTracker.updateElapsedJiffies(processTotalJiffies,
|
cpuTimeTracker.updateElapsedJiffies(processTotalJiffies,
|
||||||
clock.getTime());
|
clock.getTime());
|
||||||
return cpuTimeTracker.getCpuTrackerUsagePercent();
|
return cpuTimeTracker.getCpuTrackerUsagePercent();
|
||||||
|
@ -793,9 +788,7 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
|
||||||
if (memInfo.find()) {
|
if (memInfo.find()) {
|
||||||
String key = memInfo.group(1).trim();
|
String key = memInfo.group(1).trim();
|
||||||
String value = memInfo.group(2).replace(KB, "").trim();
|
String value = memInfo.group(2).replace(KB, "").trim();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("MemInfo : {} : Value : {}", key, value);
|
||||||
LOG.debug("MemInfo : " + key + " : Value : " + value);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (memoryMappingInfo != null) {
|
if (memoryMappingInfo != null) {
|
||||||
memoryMappingInfo.setMemInfo(key, value);
|
memoryMappingInfo.setMemInfo(key, value);
|
||||||
|
@ -941,9 +934,7 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree {
|
||||||
if (info == null) {
|
if (info == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("setMemInfo : memInfo : {}", info);
|
||||||
LOG.debug("setMemInfo : memInfo : " + info);
|
|
||||||
}
|
|
||||||
switch (info) {
|
switch (info) {
|
||||||
case SIZE:
|
case SIZE:
|
||||||
size = val;
|
size = val;
|
||||||
|
|
|
@ -133,11 +133,11 @@ public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree {
|
||||||
pInfo.cpuTimeMs = Long.parseLong(procInfo[3]);
|
pInfo.cpuTimeMs = Long.parseLong(procInfo[3]);
|
||||||
allProcs.put(pInfo.pid, pInfo);
|
allProcs.put(pInfo.pid, pInfo);
|
||||||
} catch (NumberFormatException nfe) {
|
} catch (NumberFormatException nfe) {
|
||||||
LOG.debug("Error parsing procInfo." + nfe);
|
LOG.debug("Error parsing procInfo.", nfe);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
LOG.debug("Expected split length of proc info to be "
|
LOG.debug("Expected split length of proc info to be {}. Got {}",
|
||||||
+ procInfoSplitCount + ". Got " + procInfo.length);
|
procInfoSplitCount, procInfo.length);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -102,7 +102,7 @@ public class YarnVersionInfo extends VersionInfo {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
LOG.debug("version: "+ getVersion());
|
LOG.debug("version: {}", getVersion());
|
||||||
System.out.println("YARN " + getVersion());
|
System.out.println("YARN " + getVersion());
|
||||||
System.out.println("Subversion " + getUrl() + " -r " + getRevision());
|
System.out.println("Subversion " + getUrl() + " -r " + getRevision());
|
||||||
System.out.println("Compiled by " + getUser() + " on " + getDate());
|
System.out.println("Compiled by " + getUser() + " on " + getDate());
|
||||||
|
|
|
@ -271,7 +271,7 @@ public class TestProcfsBasedProcessTree {
|
||||||
fReader = new FileReader(pidFileName);
|
fReader = new FileReader(pidFileName);
|
||||||
pidFile = new BufferedReader(fReader);
|
pidFile = new BufferedReader(fReader);
|
||||||
} catch (FileNotFoundException f) {
|
} catch (FileNotFoundException f) {
|
||||||
LOG.debug("PidFile doesn't exist : " + pidFileName);
|
LOG.debug("PidFile doesn't exist : {}", pidFileName);
|
||||||
return pid;
|
return pid;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -99,16 +99,12 @@ public class DefaultCsiAdaptorImpl implements CsiAdaptorPlugin {
|
||||||
@Override
|
@Override
|
||||||
public NodePublishVolumeResponse nodePublishVolume(
|
public NodePublishVolumeResponse nodePublishVolume(
|
||||||
NodePublishVolumeRequest request) throws YarnException, IOException {
|
NodePublishVolumeRequest request) throws YarnException, IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Received nodePublishVolume call, request: {}",
|
||||||
LOG.debug("Received nodePublishVolume call, request: {}",
|
request);
|
||||||
request.toString());
|
|
||||||
}
|
|
||||||
Csi.NodePublishVolumeRequest req = ProtoTranslatorFactory
|
Csi.NodePublishVolumeRequest req = ProtoTranslatorFactory
|
||||||
.getTranslator(NodePublishVolumeRequest.class,
|
.getTranslator(NodePublishVolumeRequest.class,
|
||||||
Csi.NodePublishVolumeRequest.class).convertTo(request);
|
Csi.NodePublishVolumeRequest.class).convertTo(request);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Translate to CSI proto message: {}", req);
|
||||||
LOG.debug("Translate to CSI proto message: {}", req.toString());
|
|
||||||
}
|
|
||||||
csiClient.nodePublishVolume(req);
|
csiClient.nodePublishVolume(req);
|
||||||
return NodePublishVolumeResponse.newInstance();
|
return NodePublishVolumeResponse.newInstance();
|
||||||
}
|
}
|
||||||
|
@ -116,16 +112,12 @@ public class DefaultCsiAdaptorImpl implements CsiAdaptorPlugin {
|
||||||
@Override
|
@Override
|
||||||
public NodeUnpublishVolumeResponse nodeUnpublishVolume(
|
public NodeUnpublishVolumeResponse nodeUnpublishVolume(
|
||||||
NodeUnpublishVolumeRequest request) throws YarnException, IOException {
|
NodeUnpublishVolumeRequest request) throws YarnException, IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Received nodeUnpublishVolume call, request: {}",
|
||||||
LOG.debug("Received nodeUnpublishVolume call, request: {}",
|
request);
|
||||||
request.toString());
|
|
||||||
}
|
|
||||||
Csi.NodeUnpublishVolumeRequest req = ProtoTranslatorFactory
|
Csi.NodeUnpublishVolumeRequest req = ProtoTranslatorFactory
|
||||||
.getTranslator(NodeUnpublishVolumeRequest.class,
|
.getTranslator(NodeUnpublishVolumeRequest.class,
|
||||||
Csi.NodeUnpublishVolumeRequest.class).convertTo(request);
|
Csi.NodeUnpublishVolumeRequest.class).convertTo(request);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Translate to CSI proto message: {}", req);
|
||||||
LOG.debug("Translate to CSI proto message: {}", req.toString());
|
|
||||||
}
|
|
||||||
csiClient.nodeUnpublishVolume(req);
|
csiClient.nodeUnpublishVolume(req);
|
||||||
return NodeUnpublishVolumeResponse.newInstance();
|
return NodeUnpublishVolumeResponse.newInstance();
|
||||||
}
|
}
|
||||||
|
|
|
@ -275,9 +275,7 @@ public class AHSWebServices extends WebServices {
|
||||||
try {
|
try {
|
||||||
nodeHttpAddress = getNMWebAddressFromRM(conf, nmId);
|
nodeHttpAddress = getNMWebAddressFromRM(conf, nmId);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{}", ex);
|
||||||
LOG.debug(ex.getMessage());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
|
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
|
||||||
|
@ -420,9 +418,7 @@ public class AHSWebServices extends WebServices {
|
||||||
try {
|
try {
|
||||||
nodeHttpAddress = getNMWebAddressFromRM(conf, nmId);
|
nodeHttpAddress = getNMWebAddressFromRM(conf, nmId);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{}", ex);
|
||||||
LOG.debug(ex.getMessage());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
|
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
|
||||||
|
|
|
@ -1424,9 +1424,7 @@ public class LeveldbTimelineStore extends AbstractService
|
||||||
|
|
||||||
writeBatch = db.createWriteBatch();
|
writeBatch = db.createWriteBatch();
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Deleting entity type:{} id:{}", entityType, entityId);
|
||||||
LOG.debug("Deleting entity type:" + entityType + " id:" + entityId);
|
|
||||||
}
|
|
||||||
// remove start time from cache and db
|
// remove start time from cache and db
|
||||||
writeBatch.delete(createStartTimeLookupKey(entityId, entityType));
|
writeBatch.delete(createStartTimeLookupKey(entityId, entityType));
|
||||||
EntityIdentifier entityIdentifier =
|
EntityIdentifier entityIdentifier =
|
||||||
|
@ -1452,11 +1450,8 @@ public class LeveldbTimelineStore extends AbstractService
|
||||||
Object value = GenericObjectMapper.read(key, kp.getOffset());
|
Object value = GenericObjectMapper.read(key, kp.getOffset());
|
||||||
deleteKeysWithPrefix(writeBatch, addPrimaryFilterToKey(name, value,
|
deleteKeysWithPrefix(writeBatch, addPrimaryFilterToKey(name, value,
|
||||||
deletePrefix), pfIterator);
|
deletePrefix), pfIterator);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Deleting entity type:{} id:{} primary filter entry {} {}",
|
||||||
LOG.debug("Deleting entity type:" + entityType + " id:" +
|
entityType, entityId, name, value);
|
||||||
entityId + " primary filter entry " + name + " " +
|
|
||||||
value);
|
|
||||||
}
|
|
||||||
} else if (key[prefixlen] == RELATED_ENTITIES_COLUMN[0]) {
|
} else if (key[prefixlen] == RELATED_ENTITIES_COLUMN[0]) {
|
||||||
kp = new KeyParser(key,
|
kp = new KeyParser(key,
|
||||||
prefixlen + RELATED_ENTITIES_COLUMN.length);
|
prefixlen + RELATED_ENTITIES_COLUMN.length);
|
||||||
|
@ -1471,11 +1466,9 @@ public class LeveldbTimelineStore extends AbstractService
|
||||||
}
|
}
|
||||||
writeBatch.delete(createReverseRelatedEntityKey(id, type,
|
writeBatch.delete(createReverseRelatedEntityKey(id, type,
|
||||||
relatedEntityStartTime, entityId, entityType));
|
relatedEntityStartTime, entityId, entityType));
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Deleting entity type:{} id:{} from invisible reverse"
|
||||||
LOG.debug("Deleting entity type:" + entityType + " id:" +
|
+ " related entity entry of type:{} id:{}", entityType,
|
||||||
entityId + " from invisible reverse related entity " +
|
entityId, type, id);
|
||||||
"entry of type:" + type + " id:" + id);
|
|
||||||
}
|
|
||||||
} else if (key[prefixlen] ==
|
} else if (key[prefixlen] ==
|
||||||
INVISIBLE_REVERSE_RELATED_ENTITIES_COLUMN[0]) {
|
INVISIBLE_REVERSE_RELATED_ENTITIES_COLUMN[0]) {
|
||||||
kp = new KeyParser(key, prefixlen +
|
kp = new KeyParser(key, prefixlen +
|
||||||
|
@ -1491,11 +1484,8 @@ public class LeveldbTimelineStore extends AbstractService
|
||||||
}
|
}
|
||||||
writeBatch.delete(createRelatedEntityKey(id, type,
|
writeBatch.delete(createRelatedEntityKey(id, type,
|
||||||
relatedEntityStartTime, entityId, entityType));
|
relatedEntityStartTime, entityId, entityType));
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Deleting entity type:{} id:{} from related entity entry"
|
||||||
LOG.debug("Deleting entity type:" + entityType + " id:" +
|
+" of type:{} id:{}", entityType, entityId, type, id);
|
||||||
entityId + " from related entity entry of type:" +
|
|
||||||
type + " id:" + id);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
WriteOptions writeOptions = new WriteOptions();
|
WriteOptions writeOptions = new WriteOptions();
|
||||||
|
|
|
@ -413,9 +413,7 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
|
||||||
EnumSet<Field> fields) throws IOException {
|
EnumSet<Field> fields) throws IOException {
|
||||||
Long revStartTime = getStartTimeLong(entityId, entityType);
|
Long revStartTime = getStartTimeLong(entityId, entityType);
|
||||||
if (revStartTime == null) {
|
if (revStartTime == null) {
|
||||||
if ( LOG.isDebugEnabled()) {
|
LOG.debug("Could not find start time for {} {} ", entityType, entityId);
|
||||||
LOG.debug("Could not find start time for {} {} ", entityType, entityId);
|
|
||||||
}
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
byte[] prefix = KeyBuilder.newInstance().add(entityType)
|
byte[] prefix = KeyBuilder.newInstance().add(entityType)
|
||||||
|
@ -424,9 +422,7 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
|
||||||
|
|
||||||
DB db = entitydb.getDBForStartTime(revStartTime);
|
DB db = entitydb.getDBForStartTime(revStartTime);
|
||||||
if (db == null) {
|
if (db == null) {
|
||||||
if ( LOG.isDebugEnabled()) {
|
LOG.debug("Could not find db for {} {} ", entityType, entityId);
|
||||||
LOG.debug("Could not find db for {} {} ", entityType, entityId);
|
|
||||||
}
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
try (DBIterator iterator = db.iterator()) {
|
try (DBIterator iterator = db.iterator()) {
|
||||||
|
@ -1163,9 +1159,7 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public TimelinePutResponse put(TimelineEntities entities) {
|
public TimelinePutResponse put(TimelineEntities entities) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Starting put");
|
||||||
LOG.debug("Starting put");
|
|
||||||
}
|
|
||||||
TimelinePutResponse response = new TimelinePutResponse();
|
TimelinePutResponse response = new TimelinePutResponse();
|
||||||
TreeMap<Long, RollingWriteBatch> entityUpdates =
|
TreeMap<Long, RollingWriteBatch> entityUpdates =
|
||||||
new TreeMap<Long, RollingWriteBatch>();
|
new TreeMap<Long, RollingWriteBatch>();
|
||||||
|
@ -1199,11 +1193,9 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
|
||||||
indexRollingWriteBatch.close();
|
indexRollingWriteBatch.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Put {} new leveldb entity entries and {} new leveldb index"
|
||||||
LOG.debug("Put " + entityCount + " new leveldb entity entries and "
|
+ " entries from {} timeline entities", entityCount, indexCount,
|
||||||
+ indexCount + " new leveldb index entries from "
|
entities.getEntities().size());
|
||||||
+ entities.getEntities().size() + " timeline entities");
|
|
||||||
}
|
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1521,16 +1513,11 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
|
||||||
|
|
||||||
// a large delete will hold the lock for too long
|
// a large delete will hold the lock for too long
|
||||||
if (batchSize >= writeBatchSize) {
|
if (batchSize >= writeBatchSize) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Preparing to delete a batch of {} old start times",
|
||||||
LOG.debug("Preparing to delete a batch of " + batchSize
|
batchSize);
|
||||||
+ " old start times");
|
|
||||||
}
|
|
||||||
starttimedb.write(writeBatch);
|
starttimedb.write(writeBatch);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Deleted batch of {}. Total start times deleted"
|
||||||
LOG.debug("Deleted batch of " + batchSize
|
+ " so far this cycle: {}", batchSize, startTimesCount);
|
||||||
+ ". Total start times deleted so far this cycle: "
|
|
||||||
+ startTimesCount);
|
|
||||||
}
|
|
||||||
IOUtils.cleanupWithLogger(LOG, writeBatch);
|
IOUtils.cleanupWithLogger(LOG, writeBatch);
|
||||||
writeBatch = starttimedb.createWriteBatch();
|
writeBatch = starttimedb.createWriteBatch();
|
||||||
batchSize = 0;
|
batchSize = 0;
|
||||||
|
@ -1538,16 +1525,11 @@ public class RollingLevelDBTimelineStore extends AbstractService implements
|
||||||
}
|
}
|
||||||
++totalCount;
|
++totalCount;
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Preparing to delete a batch of {} old start times",
|
||||||
LOG.debug("Preparing to delete a batch of " + batchSize
|
batchSize);
|
||||||
+ " old start times");
|
|
||||||
}
|
|
||||||
starttimedb.write(writeBatch);
|
starttimedb.write(writeBatch);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Deleted batch of {}. Total start times deleted so far"
|
||||||
LOG.debug("Deleted batch of " + batchSize
|
+ " this cycle: {}", batchSize, startTimesCount);
|
||||||
+ ". Total start times deleted so far this cycle: "
|
|
||||||
+ startTimesCount);
|
|
||||||
}
|
|
||||||
LOG.info("Deleted " + startTimesCount + "/" + totalCount
|
LOG.info("Deleted " + startTimesCount + "/" + totalCount
|
||||||
+ " start time entities earlier than " + minStartTime);
|
+ " start time entities earlier than " + minStartTime);
|
||||||
} finally {
|
} finally {
|
||||||
|
|
|
@ -127,12 +127,9 @@ public class TimelineACLsManager {
|
||||||
String owner = aclExt.owner;
|
String owner = aclExt.owner;
|
||||||
AccessControlList domainACL = aclExt.acls.get(applicationAccessType);
|
AccessControlList domainACL = aclExt.acls.get(applicationAccessType);
|
||||||
if (domainACL == null) {
|
if (domainACL == null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("ACL not found for access-type {} for domain {} owned by {}."
|
||||||
LOG.debug("ACL not found for access-type " + applicationAccessType
|
+ " Using default [{}]", applicationAccessType,
|
||||||
+ " for domain " + entity.getDomainId() + " owned by "
|
entity.getDomainId(), owner, YarnConfiguration.DEFAULT_YARN_APP_ACL);
|
||||||
+ owner + ". Using default ["
|
|
||||||
+ YarnConfiguration.DEFAULT_YARN_APP_ACL + "]");
|
|
||||||
}
|
|
||||||
domainACL =
|
domainACL =
|
||||||
new AccessControlList(YarnConfiguration.DEFAULT_YARN_APP_ACL);
|
new AccessControlList(YarnConfiguration.DEFAULT_YARN_APP_ACL);
|
||||||
}
|
}
|
||||||
|
|
|
@ -139,9 +139,7 @@ public class TimelineV1DelegationTokenSecretManagerService extends
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void storeNewMasterKey(DelegationKey key) throws IOException {
|
protected void storeNewMasterKey(DelegationKey key) throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Storing master key {}", key.getKeyId());
|
||||||
LOG.debug("Storing master key " + key.getKeyId());
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
if (stateStore != null) {
|
if (stateStore != null) {
|
||||||
stateStore.storeTokenMasterKey(key);
|
stateStore.storeTokenMasterKey(key);
|
||||||
|
@ -153,9 +151,7 @@ public class TimelineV1DelegationTokenSecretManagerService extends
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void removeStoredMasterKey(DelegationKey key) {
|
protected void removeStoredMasterKey(DelegationKey key) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Removing master key {}", key.getKeyId());
|
||||||
LOG.debug("Removing master key " + key.getKeyId());
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
if (stateStore != null) {
|
if (stateStore != null) {
|
||||||
stateStore.removeTokenMasterKey(key);
|
stateStore.removeTokenMasterKey(key);
|
||||||
|
@ -168,9 +164,7 @@ public class TimelineV1DelegationTokenSecretManagerService extends
|
||||||
@Override
|
@Override
|
||||||
protected void storeNewToken(TimelineDelegationTokenIdentifier tokenId,
|
protected void storeNewToken(TimelineDelegationTokenIdentifier tokenId,
|
||||||
long renewDate) {
|
long renewDate) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Storing token {}", tokenId.getSequenceNumber());
|
||||||
LOG.debug("Storing token " + tokenId.getSequenceNumber());
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
if (stateStore != null) {
|
if (stateStore != null) {
|
||||||
stateStore.storeToken(tokenId, renewDate);
|
stateStore.storeToken(tokenId, renewDate);
|
||||||
|
@ -183,9 +177,7 @@ public class TimelineV1DelegationTokenSecretManagerService extends
|
||||||
@Override
|
@Override
|
||||||
protected void removeStoredToken(TimelineDelegationTokenIdentifier tokenId)
|
protected void removeStoredToken(TimelineDelegationTokenIdentifier tokenId)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Storing token {}", tokenId.getSequenceNumber());
|
||||||
LOG.debug("Storing token " + tokenId.getSequenceNumber());
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
if (stateStore != null) {
|
if (stateStore != null) {
|
||||||
stateStore.removeToken(tokenId);
|
stateStore.removeToken(tokenId);
|
||||||
|
@ -198,9 +190,7 @@ public class TimelineV1DelegationTokenSecretManagerService extends
|
||||||
@Override
|
@Override
|
||||||
protected void updateStoredToken(TimelineDelegationTokenIdentifier tokenId,
|
protected void updateStoredToken(TimelineDelegationTokenIdentifier tokenId,
|
||||||
long renewDate) {
|
long renewDate) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Updating token {}", tokenId.getSequenceNumber());
|
||||||
LOG.debug("Updating token " + tokenId.getSequenceNumber());
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
if (stateStore != null) {
|
if (stateStore != null) {
|
||||||
stateStore.updateToken(tokenId, renewDate);
|
stateStore.updateToken(tokenId, renewDate);
|
||||||
|
|
|
@ -105,11 +105,9 @@ public class AMHeartbeatRequestHandler extends Thread {
|
||||||
if (request == null) {
|
if (request == null) {
|
||||||
throw new YarnException("Null allocateRequest from requestInfo");
|
throw new YarnException("Null allocateRequest from requestInfo");
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Sending Heartbeat to RM. AskList:{}",
|
||||||
LOG.debug("Sending Heartbeat to RM. AskList:"
|
((request.getAskList() == null) ? " empty" :
|
||||||
+ ((request.getAskList() == null) ? " empty"
|
request.getAskList().size()));
|
||||||
: request.getAskList().size()));
|
|
||||||
}
|
|
||||||
|
|
||||||
request.setResponseId(lastResponseId);
|
request.setResponseId(lastResponseId);
|
||||||
AllocateResponse response = rmProxyRelayer.allocate(request);
|
AllocateResponse response = rmProxyRelayer.allocate(request);
|
||||||
|
@ -125,20 +123,16 @@ public class AMHeartbeatRequestHandler extends Thread {
|
||||||
userUgi, conf);
|
userUgi, conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Received Heartbeat reply from RM. Allocated Containers:{}",
|
||||||
LOG.debug("Received Heartbeat reply from RM. Allocated Containers:"
|
((response.getAllocatedContainers() == null) ? " empty"
|
||||||
+ ((response.getAllocatedContainers() == null) ? " empty"
|
: response.getAllocatedContainers().size()));
|
||||||
: response.getAllocatedContainers().size()));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (requestInfo.getCallback() == null) {
|
if (requestInfo.getCallback() == null) {
|
||||||
throw new YarnException("Null callback from requestInfo");
|
throw new YarnException("Null callback from requestInfo");
|
||||||
}
|
}
|
||||||
requestInfo.getCallback().callback(response);
|
requestInfo.getCallback().callback(response);
|
||||||
} catch (InterruptedException ex) {
|
} catch (InterruptedException ex) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Interrupted while waiting for queue", ex);
|
||||||
LOG.debug("Interrupted while waiting for queue", ex);
|
|
||||||
}
|
|
||||||
} catch (Throwable ex) {
|
} catch (Throwable ex) {
|
||||||
LOG.warn(
|
LOG.warn(
|
||||||
"Error occurred while processing heart beat for " + applicationId,
|
"Error occurred while processing heart beat for " + applicationId,
|
||||||
|
|
|
@ -265,11 +265,8 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
|
||||||
// any cluster. Pick a random sub-cluster from active and enabled ones.
|
// any cluster. Pick a random sub-cluster from active and enabled ones.
|
||||||
targetId = getSubClusterForUnResolvedRequest(bookkeeper,
|
targetId = getSubClusterForUnResolvedRequest(bookkeeper,
|
||||||
rr.getAllocationRequestId());
|
rr.getAllocationRequestId());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("ERROR resolving sub-cluster for resourceName: {}, picked a "
|
||||||
LOG.debug("ERROR resolving sub-cluster for resourceName: "
|
+ "random subcluster to forward:{}", rr.getResourceName(), targetId);
|
||||||
+ rr.getResourceName() + ", picked a random subcluster to forward:"
|
|
||||||
+ targetId);
|
|
||||||
}
|
|
||||||
if (targetIds != null && targetIds.size() > 0) {
|
if (targetIds != null && targetIds.size() > 0) {
|
||||||
bookkeeper.addRackRR(targetId, rr);
|
bookkeeper.addRackRR(targetId, rr);
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -436,10 +436,8 @@ public class SQLFederationStateStore implements FederationStateStore {
|
||||||
"SubCluster " + subClusterId.toString() + " does not exist";
|
"SubCluster " + subClusterId.toString() + " does not exist";
|
||||||
FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
|
FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Got the information about the specified SubCluster {}",
|
||||||
LOG.debug("Got the information about the specified SubCluster "
|
subClusterInfo);
|
||||||
+ subClusterInfo.toString());
|
|
||||||
}
|
|
||||||
} catch (SQLException e) {
|
} catch (SQLException e) {
|
||||||
FederationStateStoreClientMetrics.failedStateStoreCall();
|
FederationStateStoreClientMetrics.failedStateStoreCall();
|
||||||
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
|
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
|
||||||
|
@ -700,10 +698,8 @@ public class SQLFederationStateStore implements FederationStateStore {
|
||||||
FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
|
FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Got the information about the specified application {}."
|
||||||
LOG.debug("Got the information about the specified application "
|
+ " The AM is running in {}", request.getApplicationId(), homeRM);
|
||||||
+ request.getApplicationId() + ". The AM is running in " + homeRM);
|
|
||||||
}
|
|
||||||
|
|
||||||
FederationStateStoreClientMetrics
|
FederationStateStoreClientMetrics
|
||||||
.succeededStateStoreCall(stopTime - startTime);
|
.succeededStateStoreCall(stopTime - startTime);
|
||||||
|
@ -852,10 +848,8 @@ public class SQLFederationStateStore implements FederationStateStore {
|
||||||
subClusterPolicyConfiguration =
|
subClusterPolicyConfiguration =
|
||||||
SubClusterPolicyConfiguration.newInstance(request.getQueue(),
|
SubClusterPolicyConfiguration.newInstance(request.getQueue(),
|
||||||
cstmt.getString(2), ByteBuffer.wrap(cstmt.getBytes(3)));
|
cstmt.getString(2), ByteBuffer.wrap(cstmt.getBytes(3)));
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Selected from StateStore the policy for the queue: {}",
|
||||||
LOG.debug("Selected from StateStore the policy for the queue: "
|
subClusterPolicyConfiguration);
|
||||||
+ subClusterPolicyConfiguration.toString());
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
LOG.warn("Policy for queue: {} does not exist.", request.getQueue());
|
LOG.warn("Policy for queue: {} does not exist.", request.getQueue());
|
||||||
return null;
|
return null;
|
||||||
|
|
|
@ -112,11 +112,9 @@ public class BaseContainerTokenSecretManager extends
|
||||||
protected byte[] retrievePasswordInternal(ContainerTokenIdentifier identifier,
|
protected byte[] retrievePasswordInternal(ContainerTokenIdentifier identifier,
|
||||||
MasterKeyData masterKey)
|
MasterKeyData masterKey)
|
||||||
throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
|
throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Retrieving password for {} for user {} to be run on NM {}",
|
||||||
LOG.debug("Retrieving password for {} for user {} to be run on NM {}",
|
identifier.getContainerID(), identifier.getUser(),
|
||||||
identifier.getContainerID(), identifier.getUser(),
|
identifier.getNmHostAddress());
|
||||||
identifier.getNmHostAddress());
|
|
||||||
}
|
|
||||||
return createPassword(identifier.getBytes(), masterKey.getSecretKey());
|
return createPassword(identifier.getBytes(), masterKey.getSecretKey());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -225,12 +225,12 @@ public class UnmanagedApplicationManager {
|
||||||
this.heartbeatHandler.resetLastResponseId();
|
this.heartbeatHandler.resetLastResponseId();
|
||||||
|
|
||||||
for (Container container : response.getContainersFromPreviousAttempts()) {
|
for (Container container : response.getContainersFromPreviousAttempts()) {
|
||||||
LOG.debug("RegisterUAM returned existing running container "
|
LOG.debug("RegisterUAM returned existing running container {}",
|
||||||
+ container.getId());
|
container.getId());
|
||||||
}
|
}
|
||||||
for (NMToken nmToken : response.getNMTokensFromPreviousAttempts()) {
|
for (NMToken nmToken : response.getNMTokensFromPreviousAttempts()) {
|
||||||
LOG.debug("RegisterUAM returned existing NM token for node "
|
LOG.debug("RegisterUAM returned existing NM token for node {}",
|
||||||
+ nmToken.getNodeId());
|
nmToken.getNodeId());
|
||||||
}
|
}
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"RegisterUAM returned {} existing running container and {} NM tokens",
|
"RegisterUAM returned {} existing running container and {} NM tokens",
|
||||||
|
|
|
@ -153,7 +153,7 @@ public final class YarnServerSecurityUtils {
|
||||||
credentials.readTokenStorageStream(buf);
|
credentials.readTokenStorageStream(buf);
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
for (Token<? extends TokenIdentifier> tk : credentials.getAllTokens()) {
|
for (Token<? extends TokenIdentifier> tk : credentials.getAllTokens()) {
|
||||||
LOG.debug(tk.getService() + " = " + tk.toString());
|
LOG.debug("{}={}", tk.getService(), tk);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -179,9 +179,7 @@ import java.security.PrivilegedExceptionAction;
|
||||||
nodeHttpAddress =
|
nodeHttpAddress =
|
||||||
LogWebServiceUtils.getNMWebAddressFromRM(yarnConf, nmId);
|
LogWebServiceUtils.getNMWebAddressFromRM(yarnConf, nmId);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{}", ex);
|
||||||
LOG.debug(ex.getMessage());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
|
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
|
||||||
|
@ -384,9 +382,7 @@ import java.security.PrivilegedExceptionAction;
|
||||||
nodeHttpAddress =
|
nodeHttpAddress =
|
||||||
LogWebServiceUtils.getNMWebAddressFromRM(yarnConf, nmId);
|
LogWebServiceUtils.getNMWebAddressFromRM(yarnConf, nmId);
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{}", ex);
|
||||||
LOG.debug(ex.getMessage());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
|
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) {
|
||||||
|
|
|
@ -573,10 +573,8 @@ public class DefaultContainerExecutor extends ContainerExecutor {
|
||||||
String user = ctx.getUser();
|
String user = ctx.getUser();
|
||||||
String pid = ctx.getPid();
|
String pid = ctx.getPid();
|
||||||
Signal signal = ctx.getSignal();
|
Signal signal = ctx.getSignal();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Sending signal {} to pid {} as user {}",
|
||||||
LOG.debug("Sending signal " + signal.getValue() + " to pid " + pid
|
signal.getValue(), pid, user);
|
||||||
+ " as user " + user);
|
|
||||||
}
|
|
||||||
if (!containerIsAlive(pid)) {
|
if (!containerIsAlive(pid)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
|
@ -85,11 +85,8 @@ public class DeletionService extends AbstractService {
|
||||||
|
|
||||||
public void delete(DeletionTask deletionTask) {
|
public void delete(DeletionTask deletionTask) {
|
||||||
if (debugDelay != -1) {
|
if (debugDelay != -1) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Scheduling DeletionTask (delay {}) : {}", debugDelay,
|
||||||
String msg = String.format("Scheduling DeletionTask (delay %d) : %s",
|
deletionTask);
|
||||||
debugDelay, deletionTask.toString());
|
|
||||||
LOG.debug(msg);
|
|
||||||
}
|
|
||||||
recordDeletionTaskInStateStore(deletionTask);
|
recordDeletionTaskInStateStore(deletionTask);
|
||||||
sched.schedule(deletionTask, debugDelay, TimeUnit.SECONDS);
|
sched.schedule(deletionTask, debugDelay, TimeUnit.SECONDS);
|
||||||
}
|
}
|
||||||
|
|
|
@ -314,12 +314,10 @@ public class LinuxContainerExecutor extends ContainerExecutor {
|
||||||
try {
|
try {
|
||||||
resourceHandlerChain = ResourceHandlerModule
|
resourceHandlerChain = ResourceHandlerModule
|
||||||
.getConfiguredResourceHandlerChain(conf, nmContext);
|
.getConfiguredResourceHandlerChain(conf, nmContext);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Resource handler chain enabled = {}",
|
||||||
final boolean enabled = resourceHandlerChain != null;
|
(resourceHandlerChain != null));
|
||||||
LOG.debug("Resource handler chain enabled = " + enabled);
|
|
||||||
}
|
|
||||||
if (resourceHandlerChain != null) {
|
if (resourceHandlerChain != null) {
|
||||||
LOG.debug("Bootstrapping resource handler chain: " +
|
LOG.debug("Bootstrapping resource handler chain: {}",
|
||||||
resourceHandlerChain);
|
resourceHandlerChain);
|
||||||
resourceHandlerChain.bootstrap(conf);
|
resourceHandlerChain.bootstrap(conf);
|
||||||
}
|
}
|
||||||
|
|
|
@ -200,11 +200,8 @@ public class NodeManager extends CompositeService
|
||||||
+ e.getMessage(), e);
|
+ e.getMessage(), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Distributed Node Attributes is enabled with provider class"
|
||||||
LOG.debug("Distributed Node Attributes is enabled"
|
+ " as : {}", attributesProvider.getClass());
|
||||||
+ " with provider class as : "
|
|
||||||
+ attributesProvider.getClass().toString());
|
|
||||||
}
|
|
||||||
return attributesProvider;
|
return attributesProvider;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -238,10 +235,8 @@ public class NodeManager extends CompositeService
|
||||||
"Failed to create NodeLabelsProvider : " + e.getMessage(), e);
|
"Failed to create NodeLabelsProvider : " + e.getMessage(), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Distributed Node Labels is enabled"
|
||||||
LOG.debug("Distributed Node Labels is enabled"
|
+ " with provider class as : {}", provider.getClass());
|
||||||
+ " with provider class as : " + provider.getClass().toString());
|
|
||||||
}
|
|
||||||
return provider;
|
return provider;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -617,14 +612,10 @@ public class NodeManager extends CompositeService
|
||||||
&& !ApplicationState.FINISHED.equals(app.getApplicationState())) {
|
&& !ApplicationState.FINISHED.equals(app.getApplicationState())) {
|
||||||
registeringCollectors.putIfAbsent(entry.getKey(), entry.getValue());
|
registeringCollectors.putIfAbsent(entry.getKey(), entry.getValue());
|
||||||
AppCollectorData data = entry.getValue();
|
AppCollectorData data = entry.getValue();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{} : {}@<{}, {}>", entry.getKey(), data.getCollectorAddr(),
|
||||||
LOG.debug(entry.getKey() + " : " + data.getCollectorAddr() + "@<"
|
data.getRMIdentifier(), data.getVersion());
|
||||||
+ data.getRMIdentifier() + ", " + data.getVersion() + ">");
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Remove collector data for done app {}", entry.getKey());
|
||||||
LOG.debug("Remove collector data for done app " + entry.getKey());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
knownCollectors.clear();
|
knownCollectors.clear();
|
||||||
|
|
|
@ -243,10 +243,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
|
||||||
LOG.error(message);
|
LOG.error(message);
|
||||||
throw new YarnException(message);
|
throw new YarnException(message);
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{} :{}", YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
|
||||||
LOG.debug(YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS + " :"
|
durationToTrackStoppedContainers);
|
||||||
+ durationToTrackStoppedContainers);
|
|
||||||
}
|
|
||||||
super.serviceInit(conf);
|
super.serviceInit(conf);
|
||||||
LOG.info("Initialized nodemanager with :" +
|
LOG.info("Initialized nodemanager with :" +
|
||||||
" physical-memory=" + memoryMb + " virtual-memory=" + virtualMemoryMb +
|
" physical-memory=" + memoryMb + " virtual-memory=" + virtualMemoryMb +
|
||||||
|
@ -406,10 +404,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
|
||||||
List<LogAggregationReport> logAggregationReports =
|
List<LogAggregationReport> logAggregationReports =
|
||||||
context.getNMLogAggregationStatusTracker()
|
context.getNMLogAggregationStatusTracker()
|
||||||
.pullCachedLogAggregationReports();
|
.pullCachedLogAggregationReports();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("The cache log aggregation status size:{}",
|
||||||
LOG.debug("The cache log aggregation status size:"
|
logAggregationReports.size());
|
||||||
+ logAggregationReports.size());
|
|
||||||
}
|
|
||||||
if (logAggregationReports != null
|
if (logAggregationReports != null
|
||||||
&& !logAggregationReports.isEmpty()) {
|
&& !logAggregationReports.isEmpty()) {
|
||||||
request.setLogAggregationReportsForApps(logAggregationReports);
|
request.setLogAggregationReportsForApps(logAggregationReports);
|
||||||
|
@ -519,10 +515,9 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
|
||||||
nodeHealthStatus.setIsNodeHealthy(healthChecker.isHealthy());
|
nodeHealthStatus.setIsNodeHealthy(healthChecker.isHealthy());
|
||||||
nodeHealthStatus.setLastHealthReportTime(healthChecker
|
nodeHealthStatus.setLastHealthReportTime(healthChecker
|
||||||
.getLastHealthReportTime());
|
.getLastHealthReportTime());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Node's health-status : {}, {}",
|
||||||
LOG.debug("Node's health-status : " + nodeHealthStatus.getIsNodeHealthy()
|
nodeHealthStatus.getIsNodeHealthy(),
|
||||||
+ ", " + nodeHealthStatus.getHealthReport());
|
nodeHealthStatus.getHealthReport());
|
||||||
}
|
|
||||||
List<ContainerStatus> containersStatuses = getContainerStatuses();
|
List<ContainerStatus> containersStatuses = getContainerStatuses();
|
||||||
ResourceUtilization containersUtilization = getContainersUtilization();
|
ResourceUtilization containersUtilization = getContainersUtilization();
|
||||||
ResourceUtilization nodeUtilization = getNodeUtilization();
|
ResourceUtilization nodeUtilization = getNodeUtilization();
|
||||||
|
@ -603,10 +598,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
|
||||||
container.cloneAndGetContainerStatus();
|
container.cloneAndGetContainerStatus();
|
||||||
if (containerStatus.getState() == ContainerState.COMPLETE) {
|
if (containerStatus.getState() == ContainerState.COMPLETE) {
|
||||||
if (isApplicationStopped(applicationId)) {
|
if (isApplicationStopped(applicationId)) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{} is completing, remove {} from NM context.",
|
||||||
LOG.debug(applicationId + " is completing, " + " remove "
|
applicationId, containerId);
|
||||||
+ containerId + " from NM context.");
|
|
||||||
}
|
|
||||||
context.getContainers().remove(containerId);
|
context.getContainers().remove(containerId);
|
||||||
pendingCompletedContainers.put(containerId, containerStatus);
|
pendingCompletedContainers.put(containerId, containerStatus);
|
||||||
} else {
|
} else {
|
||||||
|
@ -624,11 +617,9 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
|
||||||
}
|
}
|
||||||
|
|
||||||
containerStatuses.addAll(pendingCompletedContainers.values());
|
containerStatuses.addAll(pendingCompletedContainers.values());
|
||||||
|
LOG.debug("Sending out {} container statuses: {}",
|
||||||
|
containerStatuses.size(), containerStatuses);
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
|
||||||
LOG.debug("Sending out " + containerStatuses.size()
|
|
||||||
+ " container statuses: " + containerStatuses);
|
|
||||||
}
|
|
||||||
return containerStatuses;
|
return containerStatuses;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -815,8 +806,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
for (Map.Entry<ApplicationId, Credentials> entry : map.entrySet()) {
|
for (Map.Entry<ApplicationId, Credentials> entry : map.entrySet()) {
|
||||||
LOG.debug("Retrieved credentials form RM for " + entry.getKey() + ": "
|
LOG.debug("Retrieved credentials form RM for {}: {}",
|
||||||
+ entry.getValue().getAllTokens());
|
entry.getKey(), entry.getValue().getAllTokens());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return map;
|
return map;
|
||||||
|
@ -1126,10 +1117,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
|
||||||
NodeHeartbeatResponse response) {
|
NodeHeartbeatResponse response) {
|
||||||
if (isValueSented()) {
|
if (isValueSented()) {
|
||||||
if (response.getAreNodeAttributesAcceptedByRM()) {
|
if (response.getAreNodeAttributesAcceptedByRM()) {
|
||||||
if(LOG.isDebugEnabled()){
|
LOG.debug("Node attributes {{}} were Accepted by RM ",
|
||||||
LOG.debug("Node attributes {" + getPreviousValue()
|
getPreviousValue());
|
||||||
+ "} were Accepted by RM ");
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
// case where updated node attributes from NodeAttributesProvider
|
// case where updated node attributes from NodeAttributesProvider
|
||||||
// is sent to RM and RM rejected the attributes
|
// is sent to RM and RM rejected the attributes
|
||||||
|
@ -1279,11 +1268,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
|
||||||
NodeHeartbeatResponse response) {
|
NodeHeartbeatResponse response) {
|
||||||
if (isValueSented()) {
|
if (isValueSented()) {
|
||||||
if (response.getAreNodeLabelsAcceptedByRM()) {
|
if (response.getAreNodeLabelsAcceptedByRM()) {
|
||||||
if(LOG.isDebugEnabled()){
|
LOG.debug("Node Labels {{}} were Accepted by RM",
|
||||||
LOG.debug(
|
StringUtils.join(",", getPreviousValue()));
|
||||||
"Node Labels {" + StringUtils.join(",", getPreviousValue())
|
|
||||||
+ "} were Accepted by RM ");
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
// case where updated labels from NodeLabelsProvider is sent to RM and
|
// case where updated labels from NodeLabelsProvider is sent to RM and
|
||||||
// RM rejected the labels
|
// RM rejected the labels
|
||||||
|
@ -1410,10 +1396,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
|
||||||
Resource newResource = response.getResource();
|
Resource newResource = response.getResource();
|
||||||
if (newResource != null) {
|
if (newResource != null) {
|
||||||
updateNMResource(newResource);
|
updateNMResource(newResource);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Node's resource is updated to {}", newResource);
|
||||||
LOG.debug("Node's resource is updated to " +
|
|
||||||
newResource.toString());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (timelineServiceV2Enabled) {
|
if (timelineServiceV2Enabled) {
|
||||||
updateTimelineCollectorData(response);
|
updateTimelineCollectorData(response);
|
||||||
|
@ -1453,9 +1436,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
|
||||||
Map<ApplicationId, AppCollectorData> incomingCollectorsMap =
|
Map<ApplicationId, AppCollectorData> incomingCollectorsMap =
|
||||||
response.getAppCollectors();
|
response.getAppCollectors();
|
||||||
if (incomingCollectorsMap == null) {
|
if (incomingCollectorsMap == null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("No collectors to update RM");
|
||||||
LOG.debug("No collectors to update RM");
|
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Map<ApplicationId, AppCollectorData> knownCollectors =
|
Map<ApplicationId, AppCollectorData> knownCollectors =
|
||||||
|
@ -1472,11 +1453,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
|
||||||
// the known data (updates the known data).
|
// the known data (updates the known data).
|
||||||
AppCollectorData existingData = knownCollectors.get(appId);
|
AppCollectorData existingData = knownCollectors.get(appId);
|
||||||
if (AppCollectorData.happensBefore(existingData, collectorData)) {
|
if (AppCollectorData.happensBefore(existingData, collectorData)) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Sync a new collector address: {} for application: {}"
|
||||||
LOG.debug("Sync a new collector address: "
|
+ " from RM.", collectorData.getCollectorAddr(), appId);
|
||||||
+ collectorData.getCollectorAddr()
|
|
||||||
+ " for application: " + appId + " from RM.");
|
|
||||||
}
|
|
||||||
// Update information for clients.
|
// Update information for clients.
|
||||||
NMTimelinePublisher nmTimelinePublisher =
|
NMTimelinePublisher nmTimelinePublisher =
|
||||||
context.getNMTimelinePublisher();
|
context.getNMTimelinePublisher();
|
||||||
|
|
|
@ -247,11 +247,11 @@ public class AMRMProxyService extends CompositeService implements
|
||||||
// Retrieve the AM container credentials from NM context
|
// Retrieve the AM container credentials from NM context
|
||||||
Credentials amCred = null;
|
Credentials amCred = null;
|
||||||
for (Container container : this.nmContext.getContainers().values()) {
|
for (Container container : this.nmContext.getContainers().values()) {
|
||||||
LOG.debug("From NM Context container " + container.getContainerId());
|
LOG.debug("From NM Context container {}", container.getContainerId());
|
||||||
if (container.getContainerId().getApplicationAttemptId().equals(
|
if (container.getContainerId().getApplicationAttemptId().equals(
|
||||||
attemptId) && container.getContainerTokenIdentifier() != null) {
|
attemptId) && container.getContainerTokenIdentifier() != null) {
|
||||||
LOG.debug("Container type "
|
LOG.debug("Container type {}",
|
||||||
+ container.getContainerTokenIdentifier().getContainerType());
|
container.getContainerTokenIdentifier().getContainerType());
|
||||||
if (container.getContainerTokenIdentifier()
|
if (container.getContainerTokenIdentifier()
|
||||||
.getContainerType() == ContainerType.APPLICATION_MASTER) {
|
.getContainerType() == ContainerType.APPLICATION_MASTER) {
|
||||||
LOG.info("AM container {} found in context, has credentials: {}",
|
LOG.info("AM container {} found in context, has credentials: {}",
|
||||||
|
@ -764,9 +764,7 @@ public class AMRMProxyService extends CompositeService implements
|
||||||
AMRMProxyService.this.stopApplication(event.getApplicationID());
|
AMRMProxyService.this.stopApplication(event.getApplicationID());
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("AMRMProxy is ignoring event: {}", event.getType());
|
||||||
LOG.debug("AMRMProxy is ignoring event: " + event.getType());
|
|
||||||
}
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -248,10 +248,7 @@ public class AMRMProxyTokenSecretManager extends
|
||||||
try {
|
try {
|
||||||
ApplicationAttemptId applicationAttemptId =
|
ApplicationAttemptId applicationAttemptId =
|
||||||
identifier.getApplicationAttemptId();
|
identifier.getApplicationAttemptId();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Trying to retrieve password for {}", applicationAttemptId);
|
||||||
LOG.debug("Trying to retrieve password for "
|
|
||||||
+ applicationAttemptId);
|
|
||||||
}
|
|
||||||
if (!appAttemptSet.contains(applicationAttemptId)) {
|
if (!appAttemptSet.contains(applicationAttemptId)) {
|
||||||
throw new InvalidToken(applicationAttemptId
|
throw new InvalidToken(applicationAttemptId
|
||||||
+ " not found in AMRMProxyTokenSecretManager.");
|
+ " not found in AMRMProxyTokenSecretManager.");
|
||||||
|
|
|
@ -129,9 +129,7 @@ public final class DefaultRequestInterceptor extends
|
||||||
@Override
|
@Override
|
||||||
public AllocateResponse allocate(final AllocateRequest request)
|
public AllocateResponse allocate(final AllocateRequest request)
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Forwarding allocate request to the real YARN RM");
|
||||||
LOG.debug("Forwarding allocate request to the real YARN RM");
|
|
||||||
}
|
|
||||||
AllocateResponse allocateResponse = rmClient.allocate(request);
|
AllocateResponse allocateResponse = rmClient.allocate(request);
|
||||||
if (allocateResponse.getAMRMToken() != null) {
|
if (allocateResponse.getAMRMToken() != null) {
|
||||||
YarnServerSecurityUtils.updateAMRMToken(allocateResponse.getAMRMToken(),
|
YarnServerSecurityUtils.updateAMRMToken(allocateResponse.getAMRMToken(),
|
||||||
|
@ -161,10 +159,8 @@ public final class DefaultRequestInterceptor extends
|
||||||
public DistributedSchedulingAllocateResponse allocateForDistributedScheduling(
|
public DistributedSchedulingAllocateResponse allocateForDistributedScheduling(
|
||||||
DistributedSchedulingAllocateRequest request)
|
DistributedSchedulingAllocateRequest request)
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Forwarding allocateForDistributedScheduling request" +
|
||||||
LOG.debug("Forwarding allocateForDistributedScheduling request" +
|
"to the real YARN RM");
|
||||||
"to the real YARN RM");
|
|
||||||
}
|
|
||||||
if (getApplicationContext().getNMCotext()
|
if (getApplicationContext().getNMCotext()
|
||||||
.isDistributedSchedulingEnabled()) {
|
.isDistributedSchedulingEnabled()) {
|
||||||
DistributedSchedulingAllocateResponse allocateResponse =
|
DistributedSchedulingAllocateResponse allocateResponse =
|
||||||
|
|
|
@ -401,7 +401,7 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
|
||||||
amrmToken.decodeFromUrlString(
|
amrmToken.decodeFromUrlString(
|
||||||
new String(entry.getValue(), STRING_TO_BYTE_FORMAT));
|
new String(entry.getValue(), STRING_TO_BYTE_FORMAT));
|
||||||
uamMap.put(scId, amrmToken);
|
uamMap.put(scId, amrmToken);
|
||||||
LOG.debug("Recovered UAM in " + scId + " from NMSS");
|
LOG.debug("Recovered UAM in {} from NMSS", scId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
LOG.info("Found {} existing UAMs for application {} in NMStateStore",
|
LOG.info("Found {} existing UAMs for application {} in NMStateStore",
|
||||||
|
@ -443,8 +443,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
|
||||||
.getContainersFromPreviousAttempts()) {
|
.getContainersFromPreviousAttempts()) {
|
||||||
containerIdToSubClusterIdMap.put(container.getId(), subClusterId);
|
containerIdToSubClusterIdMap.put(container.getId(), subClusterId);
|
||||||
containers++;
|
containers++;
|
||||||
LOG.debug(" From subcluster " + subClusterId
|
LOG.debug(" From subcluster {} running container {}",
|
||||||
+ " running container " + container.getId());
|
subClusterId, container.getId());
|
||||||
}
|
}
|
||||||
LOG.info("Recovered {} running containers from UAM in {}",
|
LOG.info("Recovered {} running containers from UAM in {}",
|
||||||
response.getContainersFromPreviousAttempts().size(),
|
response.getContainersFromPreviousAttempts().size(),
|
||||||
|
@ -471,8 +471,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
|
||||||
containerIdToSubClusterIdMap.put(container.getContainerId(),
|
containerIdToSubClusterIdMap.put(container.getContainerId(),
|
||||||
this.homeSubClusterId);
|
this.homeSubClusterId);
|
||||||
containers++;
|
containers++;
|
||||||
LOG.debug(" From home RM " + this.homeSubClusterId
|
LOG.debug(" From home RM {} running container {}",
|
||||||
+ " running container " + container.getContainerId());
|
this.homeSubClusterId, container.getContainerId());
|
||||||
}
|
}
|
||||||
LOG.info("{} running containers including AM recovered from home RM {}",
|
LOG.info("{} running containers including AM recovered from home RM {}",
|
||||||
response.getContainerList().size(), this.homeSubClusterId);
|
response.getContainerList().size(), this.homeSubClusterId);
|
||||||
|
@ -797,10 +797,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
|
||||||
try {
|
try {
|
||||||
Future<FinishApplicationMasterResponseInfo> future = compSvc.take();
|
Future<FinishApplicationMasterResponseInfo> future = compSvc.take();
|
||||||
FinishApplicationMasterResponseInfo uamResponse = future.get();
|
FinishApplicationMasterResponseInfo uamResponse = future.get();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Received finish application response from RM: {}",
|
||||||
LOG.debug("Received finish application response from RM: "
|
uamResponse.getSubClusterId());
|
||||||
+ uamResponse.getSubClusterId());
|
|
||||||
}
|
|
||||||
if (uamResponse.getResponse() == null
|
if (uamResponse.getResponse() == null
|
||||||
|| !uamResponse.getResponse().getIsUnregistered()) {
|
|| !uamResponse.getResponse().getIsUnregistered()) {
|
||||||
failedToUnRegister = true;
|
failedToUnRegister = true;
|
||||||
|
|
|
@ -52,22 +52,16 @@ public final class NMProtoUtils {
|
||||||
int taskId = proto.getId();
|
int taskId = proto.getId();
|
||||||
if (proto.hasTaskType() && proto.getTaskType() != null) {
|
if (proto.hasTaskType() && proto.getTaskType() != null) {
|
||||||
if (proto.getTaskType().equals(DeletionTaskType.FILE.name())) {
|
if (proto.getTaskType().equals(DeletionTaskType.FILE.name())) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Converting recovered FileDeletionTask");
|
||||||
LOG.debug("Converting recovered FileDeletionTask");
|
|
||||||
}
|
|
||||||
return convertProtoToFileDeletionTask(proto, deletionService, taskId);
|
return convertProtoToFileDeletionTask(proto, deletionService, taskId);
|
||||||
} else if (proto.getTaskType().equals(
|
} else if (proto.getTaskType().equals(
|
||||||
DeletionTaskType.DOCKER_CONTAINER.name())) {
|
DeletionTaskType.DOCKER_CONTAINER.name())) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Converting recovered DockerContainerDeletionTask");
|
||||||
LOG.debug("Converting recovered DockerContainerDeletionTask");
|
|
||||||
}
|
|
||||||
return convertProtoToDockerContainerDeletionTask(proto, deletionService,
|
return convertProtoToDockerContainerDeletionTask(proto, deletionService,
|
||||||
taskId);
|
taskId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Unable to get task type, trying FileDeletionTask");
|
||||||
LOG.debug("Unable to get task type, trying FileDeletionTask");
|
|
||||||
}
|
|
||||||
return convertProtoToFileDeletionTask(proto, deletionService, taskId);
|
return convertProtoToFileDeletionTask(proto, deletionService, taskId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -638,9 +638,7 @@ public class AuxServices extends AbstractService
|
||||||
.getName());
|
.getName());
|
||||||
loadedAuxServices.add(service.getName());
|
loadedAuxServices.add(service.getName());
|
||||||
if (existingService != null && existingService.equals(service)) {
|
if (existingService != null && existingService.equals(service)) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Auxiliary service already loaded: {}", service.getName());
|
||||||
LOG.debug("Auxiliary service already loaded: " + service.getName());
|
|
||||||
}
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
foundChanges = true;
|
foundChanges = true;
|
||||||
|
|
|
@ -368,9 +368,7 @@ public class ContainerManagerImpl extends CompositeService implements
|
||||||
appsState.getIterator()) {
|
appsState.getIterator()) {
|
||||||
while (rasIterator.hasNext()) {
|
while (rasIterator.hasNext()) {
|
||||||
ContainerManagerApplicationProto proto = rasIterator.next();
|
ContainerManagerApplicationProto proto = rasIterator.next();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Recovering application with state: {}", proto);
|
||||||
LOG.debug("Recovering application with state: " + proto.toString());
|
|
||||||
}
|
|
||||||
recoverApplication(proto);
|
recoverApplication(proto);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -379,9 +377,7 @@ public class ContainerManagerImpl extends CompositeService implements
|
||||||
stateStore.getContainerStateIterator()) {
|
stateStore.getContainerStateIterator()) {
|
||||||
while (rcsIterator.hasNext()) {
|
while (rcsIterator.hasNext()) {
|
||||||
RecoveredContainerState rcs = rcsIterator.next();
|
RecoveredContainerState rcs = rcsIterator.next();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Recovering container with state: {}", rcs);
|
||||||
LOG.debug("Recovering container with state: " + rcs);
|
|
||||||
}
|
|
||||||
recoverContainer(rcs);
|
recoverContainer(rcs);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -428,20 +424,16 @@ public class ContainerManagerImpl extends CompositeService implements
|
||||||
FlowContextProto fcp = p.getFlowContext();
|
FlowContextProto fcp = p.getFlowContext();
|
||||||
fc = new FlowContext(fcp.getFlowName(), fcp.getFlowVersion(),
|
fc = new FlowContext(fcp.getFlowName(), fcp.getFlowVersion(),
|
||||||
fcp.getFlowRunId());
|
fcp.getFlowRunId());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug(
|
||||||
LOG.debug(
|
"Recovering Flow context: {} for an application {}", fc, appId);
|
||||||
"Recovering Flow context: " + fc + " for an application " + appId);
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
// in upgrade situations, where there is no prior existing flow context,
|
// in upgrade situations, where there is no prior existing flow context,
|
||||||
// default would be used.
|
// default would be used.
|
||||||
fc = new FlowContext(TimelineUtils.generateDefaultFlowName(null, appId),
|
fc = new FlowContext(TimelineUtils.generateDefaultFlowName(null, appId),
|
||||||
YarnConfiguration.DEFAULT_FLOW_VERSION, appId.getClusterTimestamp());
|
YarnConfiguration.DEFAULT_FLOW_VERSION, appId.getClusterTimestamp());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug(
|
||||||
LOG.debug(
|
"No prior existing flow context found. Using default Flow context: "
|
||||||
"No prior existing flow context found. Using default Flow context: "
|
+ "{} for an application {}", fc, appId);
|
||||||
+ fc + " for an application " + appId);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
LOG.info("Recovering application " + appId);
|
LOG.info("Recovering application " + appId);
|
||||||
|
@ -1206,11 +1198,8 @@ public class ContainerManagerImpl extends CompositeService implements
|
||||||
flowRunId = Long.parseLong(flowRunIdStr);
|
flowRunId = Long.parseLong(flowRunIdStr);
|
||||||
}
|
}
|
||||||
flowContext = new FlowContext(flowName, flowVersion, flowRunId);
|
flowContext = new FlowContext(flowName, flowVersion, flowRunId);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Flow context: {} created for an application {}",
|
||||||
LOG.debug(
|
flowContext, applicationID);
|
||||||
"Flow context: " + flowContext + " created for an application "
|
|
||||||
+ applicationID);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return flowContext;
|
return flowContext;
|
||||||
}
|
}
|
||||||
|
|
|
@ -639,10 +639,7 @@ public class ApplicationImpl implements Application {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
ApplicationId applicationID = event.getApplicationID();
|
ApplicationId applicationID = event.getApplicationID();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Processing {} of type {}", applicationID, event.getType());
|
||||||
LOG.debug(
|
|
||||||
"Processing " + applicationID + " of type " + event.getType());
|
|
||||||
}
|
|
||||||
ApplicationState oldState = stateMachine.getCurrentState();
|
ApplicationState oldState = stateMachine.getCurrentState();
|
||||||
ApplicationState newState = null;
|
ApplicationState newState = null;
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -2110,9 +2110,7 @@ public class ContainerImpl implements Container {
|
||||||
this.writeLock.lock();
|
this.writeLock.lock();
|
||||||
try {
|
try {
|
||||||
ContainerId containerID = event.getContainerID();
|
ContainerId containerID = event.getContainerID();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Processing {} of type {}", containerID, event.getType());
|
||||||
LOG.debug("Processing " + containerID + " of type " + event.getType());
|
|
||||||
}
|
|
||||||
ContainerState oldState = stateMachine.getCurrentState();
|
ContainerState oldState = stateMachine.getCurrentState();
|
||||||
ContainerState newState = null;
|
ContainerState newState = null;
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -52,10 +52,7 @@ public class DockerContainerDeletionTask extends DeletionTask
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Running DeletionTask : {}", this);
|
||||||
String msg = String.format("Running DeletionTask : %s", toString());
|
|
||||||
LOG.debug(msg);
|
|
||||||
}
|
|
||||||
LinuxContainerExecutor exec = ((LinuxContainerExecutor)
|
LinuxContainerExecutor exec = ((LinuxContainerExecutor)
|
||||||
getDeletionService().getContainerExecutor());
|
getDeletionService().getContainerExecutor());
|
||||||
exec.removeDockerContainer(containerId);
|
exec.removeDockerContainer(containerId);
|
||||||
|
|
|
@ -95,16 +95,11 @@ public class FileDeletionTask extends DeletionTask implements Runnable {
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Running DeletionTask : {}", this);
|
||||||
String msg = String.format("Running DeletionTask : %s", toString());
|
|
||||||
LOG.debug(msg);
|
|
||||||
}
|
|
||||||
boolean error = false;
|
boolean error = false;
|
||||||
if (null == getUser()) {
|
if (null == getUser()) {
|
||||||
if (baseDirs == null || baseDirs.size() == 0) {
|
if (baseDirs == null || baseDirs.size() == 0) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("NM deleting absolute path : {}", subDir);
|
||||||
LOG.debug("NM deleting absolute path : " + subDir);
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
lfs.delete(subDir, true);
|
lfs.delete(subDir, true);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
|
@ -114,9 +109,7 @@ public class FileDeletionTask extends DeletionTask implements Runnable {
|
||||||
} else {
|
} else {
|
||||||
for (Path baseDir : baseDirs) {
|
for (Path baseDir : baseDirs) {
|
||||||
Path del = subDir == null? baseDir : new Path(baseDir, subDir);
|
Path del = subDir == null? baseDir : new Path(baseDir, subDir);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("NM deleting path : {}", del);
|
||||||
LOG.debug("NM deleting path : " + del);
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
lfs.delete(del, true);
|
lfs.delete(del, true);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
|
@ -127,10 +120,7 @@ public class FileDeletionTask extends DeletionTask implements Runnable {
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
try {
|
try {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Deleting path: [{}] as user [{}]", subDir, getUser());
|
||||||
LOG.debug(
|
|
||||||
"Deleting path: [" + subDir + "] as user: [" + getUser() + "]");
|
|
||||||
}
|
|
||||||
if (baseDirs == null || baseDirs.size() == 0) {
|
if (baseDirs == null || baseDirs.size() == 0) {
|
||||||
getDeletionService().getContainerExecutor().deleteAsUser(
|
getDeletionService().getContainerExecutor().deleteAsUser(
|
||||||
new DeletionAsUserContext.Builder()
|
new DeletionAsUserContext.Builder()
|
||||||
|
|
|
@ -102,19 +102,14 @@ public class ContainerCleanup implements Runnable {
|
||||||
+ " No cleanup needed to be done");
|
+ " No cleanup needed to be done");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Marking container {} as inactive", containerIdStr);
|
||||||
LOG.debug("Marking container " + containerIdStr + " as inactive");
|
|
||||||
}
|
|
||||||
// this should ensure that if the container process has not launched
|
// this should ensure that if the container process has not launched
|
||||||
// by this time, it will never be launched
|
// by this time, it will never be launched
|
||||||
exec.deactivateContainer(containerId);
|
exec.deactivateContainer(containerId);
|
||||||
Path pidFilePath = launch.getPidFilePath();
|
Path pidFilePath = launch.getPidFilePath();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Getting pid for container {} to kill"
|
||||||
LOG.debug("Getting pid for container {} to kill"
|
+ " from pid file {}", containerIdStr, pidFilePath != null ?
|
||||||
+ " from pid file {}", containerIdStr, pidFilePath != null ?
|
pidFilePath : "null");
|
||||||
pidFilePath : "null");
|
|
||||||
}
|
|
||||||
|
|
||||||
// however the container process may have already started
|
// however the container process may have already started
|
||||||
try {
|
try {
|
||||||
|
|
||||||
|
@ -194,20 +189,17 @@ public class ContainerCleanup implements Runnable {
|
||||||
|
|
||||||
private void signalProcess(String processId, String user,
|
private void signalProcess(String processId, String user,
|
||||||
String containerIdStr) throws IOException {
|
String containerIdStr) throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Sending signal to pid {} as user {} for container {}",
|
||||||
LOG.debug("Sending signal to pid " + processId + " as user " + user
|
processId, user, containerIdStr);
|
||||||
+ " for container " + containerIdStr);
|
|
||||||
}
|
|
||||||
final ContainerExecutor.Signal signal =
|
final ContainerExecutor.Signal signal =
|
||||||
sleepDelayBeforeSigKill > 0 ? ContainerExecutor.Signal.TERM :
|
sleepDelayBeforeSigKill > 0 ? ContainerExecutor.Signal.TERM :
|
||||||
ContainerExecutor.Signal.KILL;
|
ContainerExecutor.Signal.KILL;
|
||||||
|
|
||||||
boolean result = sendSignal(user, processId, signal);
|
boolean result = sendSignal(user, processId, signal);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Sent signal {} to pid {} as user {} for container {},"
|
||||||
LOG.debug("Sent signal " + signal + " to pid " + processId + " as user "
|
+ " result={}", signal, processId, user, containerIdStr,
|
||||||
+ user + " for container " + containerIdStr + ", result="
|
(result ? "success" : "failed"));
|
||||||
+ (result ? "success" : "failed"));
|
|
||||||
}
|
|
||||||
if (sleepDelayBeforeSigKill > 0) {
|
if (sleepDelayBeforeSigKill > 0) {
|
||||||
new ContainerExecutor.DelayedProcessKiller(container, user, processId,
|
new ContainerExecutor.DelayedProcessKiller(container, user, processId,
|
||||||
sleepDelayBeforeSigKill, ContainerExecutor.Signal.KILL, exec).start();
|
sleepDelayBeforeSigKill, ContainerExecutor.Signal.KILL, exec).start();
|
||||||
|
@ -232,9 +224,7 @@ public class ContainerCleanup implements Runnable {
|
||||||
.setContainer(container)
|
.setContainer(container)
|
||||||
.setUser(container.getUser())
|
.setUser(container.getUser())
|
||||||
.build());
|
.build());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Sent signal to docker container {} as user {}, result={}",
|
||||||
LOG.debug("Sent signal to docker container " + containerIdStr
|
containerIdStr, user, (result ? "success" : "failed"));
|
||||||
+ " as user " + user + ", result=" + (result ? "success" : "failed"));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -647,11 +647,8 @@ public class ContainerLaunch implements Callable<Integer> {
|
||||||
|
|
||||||
protected void handleContainerExitCode(int exitCode, Path containerLogDir) {
|
protected void handleContainerExitCode(int exitCode, Path containerLogDir) {
|
||||||
ContainerId containerId = container.getContainerId();
|
ContainerId containerId = container.getContainerId();
|
||||||
|
LOG.debug("Container {} completed with exit code {}", containerId,
|
||||||
if (LOG.isDebugEnabled()) {
|
exitCode);
|
||||||
LOG.debug("Container " + containerId + " completed with exit code "
|
|
||||||
+ exitCode);
|
|
||||||
}
|
|
||||||
|
|
||||||
StringBuilder diagnosticInfo =
|
StringBuilder diagnosticInfo =
|
||||||
new StringBuilder("Container exited with a non-zero exit code ");
|
new StringBuilder("Container exited with a non-zero exit code ");
|
||||||
|
@ -840,22 +837,17 @@ public class ContainerLaunch implements Callable<Integer> {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Getting pid for container {} to send signal to from pid"
|
||||||
LOG.debug("Getting pid for container " + containerIdStr
|
+ " file {}", containerIdStr,
|
||||||
+ " to send signal to from pid file "
|
(pidFilePath != null ? pidFilePath.toString() : "null"));
|
||||||
+ (pidFilePath != null ? pidFilePath.toString() : "null"));
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// get process id from pid file if available
|
// get process id from pid file if available
|
||||||
// else if shell is still active, get it from the shell
|
// else if shell is still active, get it from the shell
|
||||||
String processId = getContainerPid();
|
String processId = getContainerPid();
|
||||||
if (processId != null) {
|
if (processId != null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Sending signal to pid {} as user {} for container {}",
|
||||||
LOG.debug("Sending signal to pid " + processId
|
processId, user, containerIdStr);
|
||||||
+ " as user " + user
|
|
||||||
+ " for container " + containerIdStr);
|
|
||||||
}
|
|
||||||
|
|
||||||
boolean result = exec.signalContainer(
|
boolean result = exec.signalContainer(
|
||||||
new ContainerSignalContext.Builder()
|
new ContainerSignalContext.Builder()
|
||||||
|
@ -1013,10 +1005,8 @@ public class ContainerLaunch implements Callable<Integer> {
|
||||||
String containerIdStr =
|
String containerIdStr =
|
||||||
container.getContainerId().toString();
|
container.getContainerId().toString();
|
||||||
String processId;
|
String processId;
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Accessing pid for container {} from pid file {}",
|
||||||
LOG.debug("Accessing pid for container " + containerIdStr
|
containerIdStr, pidFilePath);
|
||||||
+ " from pid file " + pidFilePath);
|
|
||||||
}
|
|
||||||
int sleepCounter = 0;
|
int sleepCounter = 0;
|
||||||
final int sleepInterval = 100;
|
final int sleepInterval = 100;
|
||||||
|
|
||||||
|
@ -1025,10 +1015,7 @@ public class ContainerLaunch implements Callable<Integer> {
|
||||||
while (true) {
|
while (true) {
|
||||||
processId = ProcessIdFileReader.getProcessId(pidFilePath);
|
processId = ProcessIdFileReader.getProcessId(pidFilePath);
|
||||||
if (processId != null) {
|
if (processId != null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Got pid {} for container {}", processId, containerIdStr);
|
||||||
LOG.debug(
|
|
||||||
"Got pid " + processId + " for container " + containerIdStr);
|
|
||||||
}
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
else if ((sleepCounter*sleepInterval) > maxKillWaitTime) {
|
else if ((sleepCounter*sleepInterval) > maxKillWaitTime) {
|
||||||
|
|
|
@ -464,10 +464,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
|
||||||
public String createCGroup(CGroupController controller, String cGroupId)
|
public String createCGroup(CGroupController controller, String cGroupId)
|
||||||
throws ResourceHandlerException {
|
throws ResourceHandlerException {
|
||||||
String path = getPathForCGroup(controller, cGroupId);
|
String path = getPathForCGroup(controller, cGroupId);
|
||||||
|
LOG.debug("createCgroup: {}", path);
|
||||||
if (LOG.isDebugEnabled()) {
|
|
||||||
LOG.debug("createCgroup: " + path);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!new File(path).mkdir()) {
|
if (!new File(path).mkdir()) {
|
||||||
throw new ResourceHandlerException("Failed to create cgroup at " + path);
|
throw new ResourceHandlerException("Failed to create cgroup at " + path);
|
||||||
|
@ -487,7 +484,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
|
||||||
+ "/tasks"), "UTF-8"))) {
|
+ "/tasks"), "UTF-8"))) {
|
||||||
str = inl.readLine();
|
str = inl.readLine();
|
||||||
if (str != null) {
|
if (str != null) {
|
||||||
LOG.debug("First line in cgroup tasks file: " + cgf + " " + str);
|
LOG.debug("First line in cgroup tasks file: {} {}", cgf, str);
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
LOG.warn("Failed to read cgroup tasks file. ", e);
|
LOG.warn("Failed to read cgroup tasks file. ", e);
|
||||||
|
@ -537,9 +534,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
|
||||||
boolean deleted = false;
|
boolean deleted = false;
|
||||||
String cGroupPath = getPathForCGroup(controller, cGroupId);
|
String cGroupPath = getPathForCGroup(controller, cGroupId);
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("deleteCGroup: {}", cGroupPath);
|
||||||
LOG.debug("deleteCGroup: " + cGroupPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
long start = clock.getTime();
|
long start = clock.getTime();
|
||||||
|
|
||||||
|
|
|
@ -153,9 +153,7 @@ public class NetworkPacketTaggingHandlerImpl
|
||||||
@Override
|
@Override
|
||||||
public List<PrivilegedOperation> teardown()
|
public List<PrivilegedOperation> teardown()
|
||||||
throws ResourceHandlerException {
|
throws ResourceHandlerException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("teardown(): Nothing to do");
|
||||||
LOG.debug("teardown(): Nothing to do");
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
|
@ -84,9 +84,7 @@ public class ResourceHandlerModule {
|
||||||
if (cGroupsHandler == null) {
|
if (cGroupsHandler == null) {
|
||||||
cGroupsHandler = new CGroupsHandlerImpl(conf,
|
cGroupsHandler = new CGroupsHandlerImpl(conf,
|
||||||
PrivilegedOperationExecutor.getInstance(conf));
|
PrivilegedOperationExecutor.getInstance(conf));
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Value of CGroupsHandler is: {}", cGroupsHandler);
|
||||||
LOG.debug("Value of CGroupsHandler is: " + cGroupsHandler);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -318,16 +316,12 @@ public class ResourceHandlerModule {
|
||||||
|
|
||||||
Map<String, ResourcePlugin> pluginMap = pluginManager.getNameToPlugins();
|
Map<String, ResourcePlugin> pluginMap = pluginManager.getNameToPlugins();
|
||||||
if (pluginMap == null) {
|
if (pluginMap == null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("List of plugins of ResourcePluginManager was empty " +
|
||||||
LOG.debug("List of plugins of ResourcePluginManager was empty " +
|
"while trying to add ResourceHandlers from configuration!");
|
||||||
"while trying to add ResourceHandlers from configuration!");
|
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
} else {
|
} else {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("List of plugins of ResourcePluginManager: {}",
|
||||||
LOG.debug("List of plugins of ResourcePluginManager: " +
|
pluginManager.getNameToPlugins());
|
||||||
pluginManager.getNameToPlugins());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for (ResourcePlugin plugin : pluginMap.values()) {
|
for (ResourcePlugin plugin : pluginMap.values()) {
|
||||||
|
|
|
@ -185,10 +185,8 @@ public class TrafficControlBandwidthHandlerImpl
|
||||||
throws ResourceHandlerException {
|
throws ResourceHandlerException {
|
||||||
String containerIdStr = containerId.toString();
|
String containerIdStr = containerId.toString();
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Attempting to reacquire classId for container: {}",
|
||||||
LOG.debug("Attempting to reacquire classId for container: " +
|
containerIdStr);
|
||||||
containerIdStr);
|
|
||||||
}
|
|
||||||
|
|
||||||
String classIdStrFromFile = cGroupsHandler.getCGroupParam(
|
String classIdStrFromFile = cGroupsHandler.getCGroupParam(
|
||||||
CGroupsHandler.CGroupController.NET_CLS, containerIdStr,
|
CGroupsHandler.CGroupController.NET_CLS, containerIdStr,
|
||||||
|
@ -277,9 +275,7 @@ public class TrafficControlBandwidthHandlerImpl
|
||||||
@Override
|
@Override
|
||||||
public List<PrivilegedOperation> teardown()
|
public List<PrivilegedOperation> teardown()
|
||||||
throws ResourceHandlerException {
|
throws ResourceHandlerException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("teardown(): Nothing to do");
|
||||||
LOG.debug("teardown(): Nothing to do");
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
|
@ -222,9 +222,7 @@ import java.util.regex.Pattern;
|
||||||
Pattern pattern = Pattern.compile(regex, Pattern.MULTILINE);
|
Pattern pattern = Pattern.compile(regex, Pattern.MULTILINE);
|
||||||
|
|
||||||
if (pattern.matcher(state).find()) {
|
if (pattern.matcher(state).find()) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Matched regex: {}", regex);
|
||||||
LOG.debug("Matched regex: " + regex);
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
String logLine = new StringBuffer("Failed to match regex: ")
|
String logLine = new StringBuffer("Failed to match regex: ")
|
||||||
.append(regex).append(" Current state: ").append(state).toString();
|
.append(regex).append(" Current state: ").append(state).toString();
|
||||||
|
@ -258,9 +256,7 @@ import java.util.regex.Pattern;
|
||||||
String output =
|
String output =
|
||||||
privilegedOperationExecutor.executePrivilegedOperation(op, true);
|
privilegedOperationExecutor.executePrivilegedOperation(op, true);
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("TC state: {}" + output);
|
||||||
LOG.debug("TC state: %n" + output);
|
|
||||||
}
|
|
||||||
|
|
||||||
return output;
|
return output;
|
||||||
} catch (PrivilegedOperationException e) {
|
} catch (PrivilegedOperationException e) {
|
||||||
|
@ -332,15 +328,11 @@ import java.util.regex.Pattern;
|
||||||
String output =
|
String output =
|
||||||
privilegedOperationExecutor.executePrivilegedOperation(op, true);
|
privilegedOperationExecutor.executePrivilegedOperation(op, true);
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("TC stats output:{}", output);
|
||||||
LOG.debug("TC stats output:" + output);
|
|
||||||
}
|
|
||||||
|
|
||||||
Map<Integer, Integer> classIdBytesStats = parseStatsString(output);
|
Map<Integer, Integer> classIdBytesStats = parseStatsString(output);
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("classId -> bytes sent {}", classIdBytesStats);
|
||||||
LOG.debug("classId -> bytes sent %n" + classIdBytesStats);
|
|
||||||
}
|
|
||||||
|
|
||||||
return classIdBytesStats;
|
return classIdBytesStats;
|
||||||
} catch (PrivilegedOperationException e) {
|
} catch (PrivilegedOperationException e) {
|
||||||
|
@ -467,9 +459,7 @@ import java.util.regex.Pattern;
|
||||||
//e.g 4325381 -> 00420005
|
//e.g 4325381 -> 00420005
|
||||||
String classIdStr = String.format("%08x", Integer.parseInt(input));
|
String classIdStr = String.format("%08x", Integer.parseInt(input));
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("ClassId hex string : {}", classIdStr);
|
||||||
LOG.debug("ClassId hex string : " + classIdStr);
|
|
||||||
}
|
|
||||||
|
|
||||||
//extract and return 4 digits
|
//extract and return 4 digits
|
||||||
//e.g 00420005 -> 0005
|
//e.g 00420005 -> 0005
|
||||||
|
|
|
@ -129,10 +129,8 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Using container runtime: {}", runtime.getClass()
|
||||||
LOG.debug("Using container runtime: " + runtime.getClass()
|
|
||||||
.getSimpleName());
|
.getSimpleName());
|
||||||
}
|
|
||||||
|
|
||||||
return runtime;
|
return runtime;
|
||||||
}
|
}
|
||||||
|
|
|
@ -511,11 +511,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
||||||
+ ", please check error message in log to understand "
|
+ ", please check error message in log to understand "
|
||||||
+ "why this happens.";
|
+ "why this happens.";
|
||||||
LOG.error(message);
|
LOG.error(message);
|
||||||
|
LOG.debug("All docker volumes in the system, command={}",
|
||||||
if (LOG.isDebugEnabled()) {
|
dockerVolumeInspectCommand);
|
||||||
LOG.debug("All docker volumes in the system, command="
|
|
||||||
+ dockerVolumeInspectCommand.toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new ContainerExecutionException(message);
|
throw new ContainerExecutionException(message);
|
||||||
}
|
}
|
||||||
|
@ -630,30 +627,22 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
||||||
protected void addCGroupParentIfRequired(String resourcesOptions,
|
protected void addCGroupParentIfRequired(String resourcesOptions,
|
||||||
String containerIdStr, DockerRunCommand runCommand) {
|
String containerIdStr, DockerRunCommand runCommand) {
|
||||||
if (cGroupsHandler == null) {
|
if (cGroupsHandler == null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("cGroupsHandler is null. cgroups are not in use. nothing to"
|
||||||
LOG.debug("cGroupsHandler is null. cgroups are not in use. nothing to"
|
|
||||||
+ " do.");
|
+ " do.");
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (resourcesOptions.equals(PrivilegedOperation.CGROUP_ARG_PREFIX
|
if (resourcesOptions.equals(PrivilegedOperation.CGROUP_ARG_PREFIX
|
||||||
+ PrivilegedOperation.CGROUP_ARG_NO_TASKS)) {
|
+ PrivilegedOperation.CGROUP_ARG_NO_TASKS)) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("no resource restrictions specified. not using docker's "
|
||||||
LOG.debug("no resource restrictions specified. not using docker's "
|
+ "cgroup options");
|
||||||
+ "cgroup options");
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("using docker's cgroups options");
|
||||||
LOG.debug("using docker's cgroups options");
|
|
||||||
}
|
|
||||||
|
|
||||||
String cGroupPath = "/"
|
String cGroupPath = "/"
|
||||||
+ cGroupsHandler.getRelativePathForCGroup(containerIdStr);
|
+ cGroupsHandler.getRelativePathForCGroup(containerIdStr);
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("using cgroup parent: {}", cGroupPath);
|
||||||
LOG.debug("using cgroup parent: " + cGroupPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
runCommand.setCGroupParent(cGroupPath);
|
runCommand.setCGroupParent(cGroupPath);
|
||||||
}
|
}
|
||||||
|
@ -1368,9 +1357,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
||||||
if (tcCommandFile != null) {
|
if (tcCommandFile != null) {
|
||||||
launchOp.appendArgs(tcCommandFile);
|
launchOp.appendArgs(tcCommandFile);
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Launching container with cmd: {}", command);
|
||||||
LOG.debug("Launching container with cmd: " + command);
|
|
||||||
}
|
|
||||||
|
|
||||||
return launchOp;
|
return launchOp;
|
||||||
}
|
}
|
||||||
|
@ -1391,8 +1378,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
||||||
throws ContainerExecutionException {
|
throws ContainerExecutionException {
|
||||||
long start = System.currentTimeMillis();
|
long start = System.currentTimeMillis();
|
||||||
DockerPullCommand dockerPullCommand = new DockerPullCommand(imageName);
|
DockerPullCommand dockerPullCommand = new DockerPullCommand(imageName);
|
||||||
LOG.debug("now pulling docker image." + " image name: " + imageName + ","
|
LOG.debug("now pulling docker image. image name: {}, container: {}",
|
||||||
+ " container: " + containerIdStr);
|
imageName, containerIdStr);
|
||||||
|
|
||||||
DockerCommandExecutor.executeDockerCommand(dockerPullCommand,
|
DockerCommandExecutor.executeDockerCommand(dockerPullCommand,
|
||||||
containerIdStr, null,
|
containerIdStr, null,
|
||||||
|
@ -1400,10 +1387,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
||||||
|
|
||||||
long end = System.currentTimeMillis();
|
long end = System.currentTimeMillis();
|
||||||
long pullImageTimeMs = end - start;
|
long pullImageTimeMs = end - start;
|
||||||
LOG.debug("pull docker image done with "
|
|
||||||
+ String.valueOf(pullImageTimeMs) + "ms spent."
|
LOG.debug("pull docker image done with {}ms specnt. image name: {},"
|
||||||
+ " image name: " + imageName + ","
|
+ " container: {}", pullImageTimeMs, imageName, containerIdStr);
|
||||||
+ " container: " + containerIdStr);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void executeLivelinessCheck(ContainerRuntimeContext ctx)
|
private void executeLivelinessCheck(ContainerRuntimeContext ctx)
|
||||||
|
|
|
@ -83,9 +83,8 @@ public final class DockerCommandExecutor {
|
||||||
if (disableFailureLogging) {
|
if (disableFailureLogging) {
|
||||||
dockerOp.disableFailureLogging();
|
dockerOp.disableFailureLogging();
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Running docker command: {}", dockerCommand);
|
||||||
LOG.debug("Running docker command: " + dockerCommand);
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
String result = privilegedOperationExecutor
|
String result = privilegedOperationExecutor
|
||||||
.executePrivilegedOperation(null, dockerOp, null,
|
.executePrivilegedOperation(null, dockerOp, null,
|
||||||
|
@ -118,17 +117,13 @@ public final class DockerCommandExecutor {
|
||||||
privilegedOperationExecutor, nmContext);
|
privilegedOperationExecutor, nmContext);
|
||||||
DockerContainerStatus dockerContainerStatus = parseContainerStatus(
|
DockerContainerStatus dockerContainerStatus = parseContainerStatus(
|
||||||
currentContainerStatus);
|
currentContainerStatus);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Container Status: {} ContainerId: {}",
|
||||||
LOG.debug("Container Status: " + dockerContainerStatus.getName()
|
dockerContainerStatus.getName(), containerId);
|
||||||
+ " ContainerId: " + containerId);
|
|
||||||
}
|
|
||||||
return dockerContainerStatus;
|
return dockerContainerStatus;
|
||||||
} catch (ContainerExecutionException e) {
|
} catch (ContainerExecutionException e) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Container Status: {} ContainerId: {}",
|
||||||
LOG.debug("Container Status: "
|
DockerContainerStatus.NONEXISTENT.getName(), containerId);
|
||||||
+ DockerContainerStatus.NONEXISTENT.getName()
|
|
||||||
+ " ContainerId: " + containerId);
|
|
||||||
}
|
|
||||||
return DockerContainerStatus.NONEXISTENT;
|
return DockerContainerStatus.NONEXISTENT;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -190,9 +190,7 @@ public class LocalizedResource implements EventHandler<ResourceEvent> {
|
||||||
this.writeLock.lock();
|
this.writeLock.lock();
|
||||||
try {
|
try {
|
||||||
Path resourcePath = event.getLocalResourceRequest().getPath();
|
Path resourcePath = event.getLocalResourceRequest().getPath();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Processing {} of type {}", resourcePath, event.getType());
|
||||||
LOG.debug("Processing " + resourcePath + " of type " + event.getType());
|
|
||||||
}
|
|
||||||
ResourceState oldState = this.stateMachine.getCurrentState();
|
ResourceState oldState = this.stateMachine.getCurrentState();
|
||||||
ResourceState newState = null;
|
ResourceState newState = null;
|
||||||
try {
|
try {
|
||||||
|
@ -201,11 +199,9 @@ public class LocalizedResource implements EventHandler<ResourceEvent> {
|
||||||
LOG.warn("Can't handle this event at current state", e);
|
LOG.warn("Can't handle this event at current state", e);
|
||||||
}
|
}
|
||||||
if (newState != null && oldState != newState) {
|
if (newState != null && oldState != newState) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Resource {}{} size : {} transitioned from {} to {}",
|
||||||
LOG.debug("Resource " + resourcePath + (localPath != null ?
|
resourcePath, (localPath != null ? "(->" + localPath + ")": ""),
|
||||||
"(->" + localPath + ")": "") + " size : " + getSize()
|
getSize(), oldState, newState);
|
||||||
+ " transitioned from " + oldState + " to " + newState);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
this.writeLock.unlock();
|
this.writeLock.unlock();
|
||||||
|
|
|
@ -345,10 +345,8 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
LocalizedResourceProto proto = it.next();
|
LocalizedResourceProto proto = it.next();
|
||||||
LocalResource rsrc = new LocalResourcePBImpl(proto.getResource());
|
LocalResource rsrc = new LocalResourcePBImpl(proto.getResource());
|
||||||
LocalResourceRequest req = new LocalResourceRequest(rsrc);
|
LocalResourceRequest req = new LocalResourceRequest(rsrc);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Recovering localized resource {} at {}",
|
||||||
LOG.debug("Recovering localized resource " + req + " at "
|
req, proto.getLocalPath());
|
||||||
+ proto.getLocalPath());
|
|
||||||
}
|
|
||||||
tracker.handle(new ResourceRecoveredEvent(req,
|
tracker.handle(new ResourceRecoveredEvent(req,
|
||||||
new Path(proto.getLocalPath()), proto.getSize()));
|
new Path(proto.getLocalPath()), proto.getSize()));
|
||||||
}
|
}
|
||||||
|
@ -514,10 +512,8 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
.getApplicationId());
|
.getApplicationId());
|
||||||
for (LocalResourceRequest req : e.getValue()) {
|
for (LocalResourceRequest req : e.getValue()) {
|
||||||
tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
|
tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Localizing {} for container {}",
|
||||||
LOG.debug("Localizing " + req.getPath() +
|
req.getPath(), c.getContainerId());
|
||||||
" for container " + c.getContainerId());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -930,17 +926,13 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
+ " Either queue is full or threadpool is shutdown.", re);
|
+ " Either queue is full or threadpool is shutdown.", re);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Skip downloading resource: {} since it's in"
|
||||||
LOG.debug("Skip downloading resource: " + key + " since it's in"
|
+ " state: {}", key, rsrc.getState());
|
||||||
+ " state: " + rsrc.getState());
|
|
||||||
}
|
|
||||||
rsrc.unlock();
|
rsrc.unlock();
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Skip downloading resource: {} since it is locked"
|
||||||
LOG.debug("Skip downloading resource: " + key + " since it is locked"
|
+ " by other threads", key);
|
||||||
+ " by other threads");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1302,10 +1294,10 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
if (systemCredentials == null) {
|
if (systemCredentials == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
|
||||||
LOG.debug("Adding new framework-token for " + appId
|
LOG.debug("Adding new framework-token for {} for localization: {}",
|
||||||
+ " for localization: " + systemCredentials.getAllTokens());
|
appId, systemCredentials.getAllTokens());
|
||||||
}
|
|
||||||
return systemCredentials;
|
return systemCredentials;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1328,11 +1320,10 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
LOG.info("Writing credentials to the nmPrivate file "
|
LOG.info("Writing credentials to the nmPrivate file "
|
||||||
+ nmPrivateCTokensPath.toString());
|
+ nmPrivateCTokensPath.toString());
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug("Credentials list in " + nmPrivateCTokensPath.toString()
|
LOG.debug("Credentials list in {}: " + nmPrivateCTokensPath);
|
||||||
+ ": ");
|
|
||||||
for (Token<? extends TokenIdentifier> tk : credentials
|
for (Token<? extends TokenIdentifier> tk : credentials
|
||||||
.getAllTokens()) {
|
.getAllTokens()) {
|
||||||
LOG.debug(tk + " : " + buildTokenFingerprint(tk));
|
LOG.debug("{} : {}", tk, buildTokenFingerprint(tk));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (UserGroupInformation.isSecurityEnabled()) {
|
if (UserGroupInformation.isSecurityEnabled()) {
|
||||||
|
|
|
@ -41,9 +41,7 @@ public class LocalizerTokenSelector implements
|
||||||
LOG.debug("Using localizerTokenSelector.");
|
LOG.debug("Using localizerTokenSelector.");
|
||||||
|
|
||||||
for (Token<? extends TokenIdentifier> token : tokens) {
|
for (Token<? extends TokenIdentifier> token : tokens) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Token of kind {} is found", token.getKind());
|
||||||
LOG.debug("Token of kind " + token.getKind() + " is found");
|
|
||||||
}
|
|
||||||
if (LocalizerTokenIdentifier.KIND.equals(token.getKind())) {
|
if (LocalizerTokenIdentifier.KIND.equals(token.getKind())) {
|
||||||
return (Token<LocalizerTokenIdentifier>) token;
|
return (Token<LocalizerTokenIdentifier>) token;
|
||||||
}
|
}
|
||||||
|
|
|
@ -383,11 +383,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
|
||||||
Credentials systemCredentials =
|
Credentials systemCredentials =
|
||||||
context.getSystemCredentialsForApps().get(appId);
|
context.getSystemCredentialsForApps().get(appId);
|
||||||
if (systemCredentials != null) {
|
if (systemCredentials != null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Adding new framework-token for {} for log-aggregation:"
|
||||||
LOG.debug("Adding new framework-token for " + appId
|
+ " {}; userUgi={}", appId, systemCredentials.getAllTokens(),
|
||||||
+ " for log-aggregation: " + systemCredentials.getAllTokens()
|
userUgi);
|
||||||
+ "; userUgi=" + userUgi);
|
|
||||||
}
|
|
||||||
// this will replace old token
|
// this will replace old token
|
||||||
userUgi.addCredentials(systemCredentials);
|
userUgi.addCredentials(systemCredentials);
|
||||||
}
|
}
|
||||||
|
|
|
@ -132,10 +132,8 @@ public class NonAggregatingLogHandler extends AbstractService implements
|
||||||
ApplicationId appId = entry.getKey();
|
ApplicationId appId = entry.getKey();
|
||||||
LogDeleterProto proto = entry.getValue();
|
LogDeleterProto proto = entry.getValue();
|
||||||
long deleteDelayMsec = proto.getDeletionTime() - now;
|
long deleteDelayMsec = proto.getDeletionTime() - now;
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Scheduling deletion of {} logs in {} msec", appId,
|
||||||
LOG.debug("Scheduling deletion of " + appId + " logs in "
|
deleteDelayMsec);
|
||||||
+ deleteDelayMsec + " msec");
|
|
||||||
}
|
|
||||||
LogDeleterRunnable logDeleter =
|
LogDeleterRunnable logDeleter =
|
||||||
new LogDeleterRunnable(proto.getUser(), appId);
|
new LogDeleterRunnable(proto.getUser(), appId);
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -468,8 +468,8 @@ public class ContainersMonitorImpl extends AbstractService implements
|
||||||
tmp.append(p.getPID());
|
tmp.append(p.getPID());
|
||||||
tmp.append(" ");
|
tmp.append(" ");
|
||||||
}
|
}
|
||||||
LOG.debug("Current ProcessTree list : "
|
LOG.debug("Current ProcessTree list : {}",
|
||||||
+ tmp.substring(0, tmp.length()) + "]");
|
tmp.substring(0, tmp.length()) + "]");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Temporary structure to calculate the total resource utilization of
|
// Temporary structure to calculate the total resource utilization of
|
||||||
|
@ -495,10 +495,8 @@ public class ContainersMonitorImpl extends AbstractService implements
|
||||||
if (pId == null || !isResourceCalculatorAvailable()) {
|
if (pId == null || !isResourceCalculatorAvailable()) {
|
||||||
continue; // processTree cannot be tracked
|
continue; // processTree cannot be tracked
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Constructing ProcessTree for : PID = {}"
|
||||||
LOG.debug("Constructing ProcessTree for : PID = " + pId
|
+" ContainerId = {}", pId, containerId);
|
||||||
+ " ContainerId = " + containerId);
|
|
||||||
}
|
|
||||||
ResourceCalculatorProcessTree pTree = ptInfo.getProcessTree();
|
ResourceCalculatorProcessTree pTree = ptInfo.getProcessTree();
|
||||||
pTree.updateProcessTree(); // update process-tree
|
pTree.updateProcessTree(); // update process-tree
|
||||||
long currentVmemUsage = pTree.getVirtualMemorySize();
|
long currentVmemUsage = pTree.getVirtualMemorySize();
|
||||||
|
@ -536,13 +534,11 @@ public class ContainersMonitorImpl extends AbstractService implements
|
||||||
+ "while monitoring resource of {}", containerId, e);
|
+ "while monitoring resource of {}", containerId, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Total Resource Usage stats in NM by all containers : "
|
||||||
LOG.debug("Total Resource Usage stats in NM by all containers : "
|
+ "Virtual Memory= {}, Physical Memory= {}, "
|
||||||
+ "Virtual Memory= " + vmemUsageByAllContainers
|
+ "Total CPU usage(% per core)= {}", vmemUsageByAllContainers,
|
||||||
+ ", Physical Memory= " + pmemByAllContainers
|
pmemByAllContainers, cpuUsagePercentPerCoreByAllContainers);
|
||||||
+ ", Total CPU usage(% per core)= "
|
|
||||||
+ cpuUsagePercentPerCoreByAllContainers);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Save the aggregated utilization of the containers
|
// Save the aggregated utilization of the containers
|
||||||
setContainersUtilization(trackedContainersUtilization);
|
setContainersUtilization(trackedContainersUtilization);
|
||||||
|
@ -587,9 +583,7 @@ public class ContainersMonitorImpl extends AbstractService implements
|
||||||
if (pId != null) {
|
if (pId != null) {
|
||||||
// pId will be null, either if the container is not spawned yet
|
// pId will be null, either if the container is not spawned yet
|
||||||
// or if the container's pid is removed from ContainerExecutor
|
// or if the container's pid is removed from ContainerExecutor
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Tracking ProcessTree {} for the first time", pId);
|
||||||
LOG.debug("Tracking ProcessTree " + pId + " for the first time");
|
|
||||||
}
|
|
||||||
ResourceCalculatorProcessTree pt =
|
ResourceCalculatorProcessTree pt =
|
||||||
getResourceCalculatorProcessTree(pId);
|
getResourceCalculatorProcessTree(pId);
|
||||||
ptInfo.setPid(pId);
|
ptInfo.setPid(pId);
|
||||||
|
|
|
@ -159,9 +159,7 @@ public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin,
|
||||||
lastTimeFoundDevices = r;
|
lastTimeFoundDevices = r;
|
||||||
return r;
|
return r;
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Failed to get output from {}", pathOfGpuBinary);
|
||||||
LOG.debug("Failed to get output from " + pathOfGpuBinary);
|
|
||||||
}
|
|
||||||
throw new YarnException(e);
|
throw new YarnException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -169,10 +167,8 @@ public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin,
|
||||||
@Override
|
@Override
|
||||||
public DeviceRuntimeSpec onDevicesAllocated(Set<Device> allocatedDevices,
|
public DeviceRuntimeSpec onDevicesAllocated(Set<Device> allocatedDevices,
|
||||||
YarnRuntimeType yarnRuntime) throws Exception {
|
YarnRuntimeType yarnRuntime) throws Exception {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Generating runtime spec for allocated devices: {}, {}",
|
||||||
LOG.debug("Generating runtime spec for allocated devices: "
|
allocatedDevices, yarnRuntime.getName());
|
||||||
+ allocatedDevices + ", " + yarnRuntime.getName());
|
|
||||||
}
|
|
||||||
if (yarnRuntime == YarnRuntimeType.RUNTIME_DOCKER) {
|
if (yarnRuntime == YarnRuntimeType.RUNTIME_DOCKER) {
|
||||||
String nvidiaRuntime = "nvidia";
|
String nvidiaRuntime = "nvidia";
|
||||||
String nvidiaVisibleDevices = "NVIDIA_VISIBLE_DEVICES";
|
String nvidiaVisibleDevices = "NVIDIA_VISIBLE_DEVICES";
|
||||||
|
@ -201,14 +197,10 @@ public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin,
|
||||||
String output = null;
|
String output = null;
|
||||||
// output "major:minor" in hex
|
// output "major:minor" in hex
|
||||||
try {
|
try {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Get major numbers from /dev/{}", devName);
|
||||||
LOG.debug("Get major numbers from /dev/" + devName);
|
|
||||||
}
|
|
||||||
output = shellExecutor.getMajorMinorInfo(devName);
|
output = shellExecutor.getMajorMinorInfo(devName);
|
||||||
String[] strs = output.trim().split(":");
|
String[] strs = output.trim().split(":");
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("stat output:{}", output);
|
||||||
LOG.debug("stat output:" + output);
|
|
||||||
}
|
|
||||||
output = Integer.toString(Integer.parseInt(strs[0], 16));
|
output = Integer.toString(Integer.parseInt(strs[0], 16));
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
String msg =
|
String msg =
|
||||||
|
|
|
@ -164,10 +164,10 @@ public class IntelFpgaOpenclPlugin implements AbstractFpgaVendorPlugin {
|
||||||
Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
|
Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(
|
||||||
new String[]{"stat", "-c", "%t:%T", "/dev/" + devName});
|
new String[]{"stat", "-c", "%t:%T", "/dev/" + devName});
|
||||||
try {
|
try {
|
||||||
LOG.debug("Get FPGA major-minor numbers from /dev/" + devName);
|
LOG.debug("Get FPGA major-minor numbers from /dev/{}", devName);
|
||||||
shexec.execute();
|
shexec.execute();
|
||||||
String[] strs = shexec.getOutput().trim().split(":");
|
String[] strs = shexec.getOutput().trim().split(":");
|
||||||
LOG.debug("stat output:" + shexec.getOutput());
|
LOG.debug("stat output:{}", shexec.getOutput());
|
||||||
output = Integer.parseInt(strs[0], 16) + ":" +
|
output = Integer.parseInt(strs[0], 16) + ":" +
|
||||||
Integer.parseInt(strs[1], 16);
|
Integer.parseInt(strs[1], 16);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
|
@ -192,7 +192,7 @@ public class IntelFpgaOpenclPlugin implements AbstractFpgaVendorPlugin {
|
||||||
"Failed to execute " + binary + " diagnose, exception message:" + e
|
"Failed to execute " + binary + " diagnose, exception message:" + e
|
||||||
.getMessage() +", output:" + output + ", continue ...";
|
.getMessage() +", output:" + output + ", continue ...";
|
||||||
LOG.warn(msg);
|
LOG.warn(msg);
|
||||||
LOG.debug(shexec.getOutput());
|
LOG.debug("{}", shexec.getOutput());
|
||||||
}
|
}
|
||||||
return shexec.getOutput();
|
return shexec.getOutput();
|
||||||
}
|
}
|
||||||
|
@ -241,7 +241,7 @@ public class IntelFpgaOpenclPlugin implements AbstractFpgaVendorPlugin {
|
||||||
|
|
||||||
if (aocxPath.isPresent()) {
|
if (aocxPath.isPresent()) {
|
||||||
ipFilePath = aocxPath.get().toUri().toString();
|
ipFilePath = aocxPath.get().toUri().toString();
|
||||||
LOG.debug("Found: " + ipFilePath);
|
LOG.debug("Found: {}", ipFilePath);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
LOG.warn("Localized resource is null!");
|
LOG.warn("Localized resource is null!");
|
||||||
|
@ -278,7 +278,7 @@ public class IntelFpgaOpenclPlugin implements AbstractFpgaVendorPlugin {
|
||||||
try {
|
try {
|
||||||
shexec.execute();
|
shexec.execute();
|
||||||
if (0 == shexec.getExitCode()) {
|
if (0 == shexec.getExitCode()) {
|
||||||
LOG.debug(shexec.getOutput());
|
LOG.debug("{}", shexec.getOutput());
|
||||||
LOG.info("Intel aocl program " + ipPath + " to " +
|
LOG.info("Intel aocl program " + ipPath + " to " +
|
||||||
aclName + " successfully");
|
aclName + " successfully");
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -129,9 +129,7 @@ public class GpuDiscoverer {
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
numOfErrorExecutionSinceLastSucceed++;
|
numOfErrorExecutionSinceLastSucceed++;
|
||||||
String msg = getErrorMessageOfScriptExecution(e.getMessage());
|
String msg = getErrorMessageOfScriptExecution(e.getMessage());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug(msg);
|
||||||
LOG.debug(msg);
|
|
||||||
}
|
|
||||||
throw new YarnException(msg, e);
|
throw new YarnException(msg, e);
|
||||||
} catch (YarnException e) {
|
} catch (YarnException e) {
|
||||||
numOfErrorExecutionSinceLastSucceed++;
|
numOfErrorExecutionSinceLastSucceed++;
|
||||||
|
|
|
@ -118,11 +118,9 @@ public class AllocationBasedResourceUtilizationTracker implements
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("before cpuCheck [asked={} > allowed={}]",
|
||||||
LOG.debug("before cpuCheck [asked={} > allowed={}]",
|
this.containersAllocation.getCPU(),
|
||||||
this.containersAllocation.getCPU(),
|
getContainersMonitor().getVCoresAllocatedForContainers());
|
||||||
getContainersMonitor().getVCoresAllocatedForContainers());
|
|
||||||
}
|
|
||||||
// Check CPU.
|
// Check CPU.
|
||||||
if (this.containersAllocation.getCPU() + cpuVcores >
|
if (this.containersAllocation.getCPU() + cpuVcores >
|
||||||
getContainersMonitor().getVCoresAllocatedForContainers()) {
|
getContainersMonitor().getVCoresAllocatedForContainers()) {
|
||||||
|
|
|
@ -137,10 +137,8 @@ public class ContainerScheduler extends AbstractService implements
|
||||||
resourceHandlerChain = ResourceHandlerModule
|
resourceHandlerChain = ResourceHandlerModule
|
||||||
.getConfiguredResourceHandlerChain(conf, context);
|
.getConfiguredResourceHandlerChain(conf, context);
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Resource handler chain enabled = {}",
|
||||||
LOG.debug("Resource handler chain enabled = " + (resourceHandlerChain
|
(resourceHandlerChain != null));
|
||||||
!= null));
|
|
||||||
}
|
|
||||||
if (resourceHandlerChain != null) {
|
if (resourceHandlerChain != null) {
|
||||||
LOG.debug("Bootstrapping resource handler chain");
|
LOG.debug("Bootstrapping resource handler chain");
|
||||||
resourceHandlerChain.bootstrap(conf);
|
resourceHandlerChain.bootstrap(conf);
|
||||||
|
|
|
@ -447,10 +447,8 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
public void storeContainer(ContainerId containerId, int containerVersion,
|
public void storeContainer(ContainerId containerId, int containerVersion,
|
||||||
long startTime, StartContainerRequest startRequest) throws IOException {
|
long startTime, StartContainerRequest startRequest) throws IOException {
|
||||||
String idStr = containerId.toString();
|
String idStr = containerId.toString();
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("storeContainer: containerId= {}, startRequest= {}",
|
||||||
LOG.debug("storeContainer: containerId= " + idStr
|
idStr, startRequest);
|
||||||
+ ", startRequest= " + startRequest);
|
|
||||||
}
|
|
||||||
final String keyVersion = getContainerVersionKey(idStr);
|
final String keyVersion = getContainerVersionKey(idStr);
|
||||||
final String keyRequest =
|
final String keyRequest =
|
||||||
getContainerKey(idStr, CONTAINER_REQUEST_KEY_SUFFIX);
|
getContainerKey(idStr, CONTAINER_REQUEST_KEY_SUFFIX);
|
||||||
|
@ -488,9 +486,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void storeContainerQueued(ContainerId containerId) throws IOException {
|
public void storeContainerQueued(ContainerId containerId) throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("storeContainerQueued: containerId={}", containerId);
|
||||||
LOG.debug("storeContainerQueued: containerId=" + containerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
||||||
+ CONTAINER_QUEUED_KEY_SUFFIX;
|
+ CONTAINER_QUEUED_KEY_SUFFIX;
|
||||||
|
@ -504,9 +500,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
|
|
||||||
private void removeContainerQueued(ContainerId containerId)
|
private void removeContainerQueued(ContainerId containerId)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("removeContainerQueued: containerId={}", containerId);
|
||||||
LOG.debug("removeContainerQueued: containerId=" + containerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
||||||
+ CONTAINER_QUEUED_KEY_SUFFIX;
|
+ CONTAINER_QUEUED_KEY_SUFFIX;
|
||||||
|
@ -520,9 +514,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void storeContainerPaused(ContainerId containerId) throws IOException {
|
public void storeContainerPaused(ContainerId containerId) throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("storeContainerPaused: containerId={}", containerId);
|
||||||
LOG.debug("storeContainerPaused: containerId=" + containerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
||||||
+ CONTAINER_PAUSED_KEY_SUFFIX;
|
+ CONTAINER_PAUSED_KEY_SUFFIX;
|
||||||
|
@ -537,9 +529,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
@Override
|
@Override
|
||||||
public void removeContainerPaused(ContainerId containerId)
|
public void removeContainerPaused(ContainerId containerId)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("removeContainerPaused: containerId={}", containerId);
|
||||||
LOG.debug("removeContainerPaused: containerId=" + containerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
||||||
+ CONTAINER_PAUSED_KEY_SUFFIX;
|
+ CONTAINER_PAUSED_KEY_SUFFIX;
|
||||||
|
@ -554,10 +544,8 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
@Override
|
@Override
|
||||||
public void storeContainerDiagnostics(ContainerId containerId,
|
public void storeContainerDiagnostics(ContainerId containerId,
|
||||||
StringBuilder diagnostics) throws IOException {
|
StringBuilder diagnostics) throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("storeContainerDiagnostics: containerId={}, diagnostics=",
|
||||||
LOG.debug("storeContainerDiagnostics: containerId=" + containerId
|
containerId, diagnostics);
|
||||||
+ ", diagnostics=" + diagnostics);
|
|
||||||
}
|
|
||||||
|
|
||||||
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
||||||
+ CONTAINER_DIAGS_KEY_SUFFIX;
|
+ CONTAINER_DIAGS_KEY_SUFFIX;
|
||||||
|
@ -572,9 +560,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
@Override
|
@Override
|
||||||
public void storeContainerLaunched(ContainerId containerId)
|
public void storeContainerLaunched(ContainerId containerId)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("storeContainerLaunched: containerId={}", containerId);
|
||||||
LOG.debug("storeContainerLaunched: containerId=" + containerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Removing the container if queued for backward compatibility reasons
|
// Removing the container if queued for backward compatibility reasons
|
||||||
removeContainerQueued(containerId);
|
removeContainerQueued(containerId);
|
||||||
|
@ -591,9 +577,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
@Override
|
@Override
|
||||||
public void storeContainerUpdateToken(ContainerId containerId,
|
public void storeContainerUpdateToken(ContainerId containerId,
|
||||||
ContainerTokenIdentifier containerTokenIdentifier) throws IOException {
|
ContainerTokenIdentifier containerTokenIdentifier) throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("storeContainerUpdateToken: containerId={}", containerId);
|
||||||
LOG.debug("storeContainerUpdateToken: containerId=" + containerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
String keyUpdateToken = CONTAINERS_KEY_PREFIX + containerId.toString()
|
String keyUpdateToken = CONTAINERS_KEY_PREFIX + containerId.toString()
|
||||||
+ CONTAINER_UPDATE_TOKEN_SUFFIX;
|
+ CONTAINER_UPDATE_TOKEN_SUFFIX;
|
||||||
|
@ -621,9 +605,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
@Override
|
@Override
|
||||||
public void storeContainerKilled(ContainerId containerId)
|
public void storeContainerKilled(ContainerId containerId)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("storeContainerKilled: containerId={}", containerId);
|
||||||
LOG.debug("storeContainerKilled: containerId=" + containerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
||||||
+ CONTAINER_KILLED_KEY_SUFFIX;
|
+ CONTAINER_KILLED_KEY_SUFFIX;
|
||||||
|
@ -638,9 +620,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
@Override
|
@Override
|
||||||
public void storeContainerCompleted(ContainerId containerId,
|
public void storeContainerCompleted(ContainerId containerId,
|
||||||
int exitCode) throws IOException {
|
int exitCode) throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("storeContainerCompleted: containerId={}", containerId);
|
||||||
LOG.debug("storeContainerCompleted: containerId=" + containerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
String key = CONTAINERS_KEY_PREFIX + containerId.toString()
|
||||||
+ CONTAINER_EXIT_CODE_KEY_SUFFIX;
|
+ CONTAINER_EXIT_CODE_KEY_SUFFIX;
|
||||||
|
@ -706,9 +686,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
@Override
|
@Override
|
||||||
public void removeContainer(ContainerId containerId)
|
public void removeContainer(ContainerId containerId)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("removeContainer: containerId={}", containerId);
|
||||||
LOG.debug("removeContainer: containerId=" + containerId);
|
|
||||||
}
|
|
||||||
|
|
||||||
String keyPrefix = CONTAINERS_KEY_PREFIX + containerId.toString();
|
String keyPrefix = CONTAINERS_KEY_PREFIX + containerId.toString();
|
||||||
try {
|
try {
|
||||||
|
@ -789,10 +767,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
@Override
|
@Override
|
||||||
public void storeApplication(ApplicationId appId,
|
public void storeApplication(ApplicationId appId,
|
||||||
ContainerManagerApplicationProto p) throws IOException {
|
ContainerManagerApplicationProto p) throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("storeApplication: appId={}, proto={}", appId, p);
|
||||||
LOG.debug("storeApplication: appId=" + appId
|
|
||||||
+ ", proto=" + p);
|
|
||||||
}
|
|
||||||
|
|
||||||
String key = APPLICATIONS_KEY_PREFIX + appId;
|
String key = APPLICATIONS_KEY_PREFIX + appId;
|
||||||
try {
|
try {
|
||||||
|
@ -806,9 +781,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
@Override
|
@Override
|
||||||
public void removeApplication(ApplicationId appId)
|
public void removeApplication(ApplicationId appId)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("removeApplication: appId={}", appId);
|
||||||
LOG.debug("removeApplication: appId=" + appId);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
WriteBatch batch = db.createWriteBatch();
|
WriteBatch batch = db.createWriteBatch();
|
||||||
|
@ -917,9 +890,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Loading completed resource from {}", key);
|
||||||
LOG.debug("Loading completed resource from " + key);
|
|
||||||
}
|
|
||||||
nextCompletedResource = LocalizedResourceProto.parseFrom(
|
nextCompletedResource = LocalizedResourceProto.parseFrom(
|
||||||
entry.getValue());
|
entry.getValue());
|
||||||
}
|
}
|
||||||
|
@ -952,9 +923,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
}
|
}
|
||||||
|
|
||||||
Path localPath = new Path(key.substring(keyPrefix.length()));
|
Path localPath = new Path(key.substring(keyPrefix.length()));
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Loading in-progress resource at {}", localPath);
|
||||||
LOG.debug("Loading in-progress resource at " + localPath);
|
|
||||||
}
|
|
||||||
nextStartedResource = new SimpleEntry<LocalResourceProto, Path>(
|
nextStartedResource = new SimpleEntry<LocalResourceProto, Path>(
|
||||||
LocalResourceProto.parseFrom(entry.getValue()), localPath);
|
LocalResourceProto.parseFrom(entry.getValue()), localPath);
|
||||||
}
|
}
|
||||||
|
@ -1042,9 +1011,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
String localPath = proto.getLocalPath();
|
String localPath = proto.getLocalPath();
|
||||||
String startedKey = getResourceStartedKey(user, appId, localPath);
|
String startedKey = getResourceStartedKey(user, appId, localPath);
|
||||||
String completedKey = getResourceCompletedKey(user, appId, localPath);
|
String completedKey = getResourceCompletedKey(user, appId, localPath);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Storing localized resource to {}", completedKey);
|
||||||
LOG.debug("Storing localized resource to " + completedKey);
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
WriteBatch batch = db.createWriteBatch();
|
WriteBatch batch = db.createWriteBatch();
|
||||||
try {
|
try {
|
||||||
|
@ -1066,9 +1033,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
String localPathStr = localPath.toString();
|
String localPathStr = localPath.toString();
|
||||||
String startedKey = getResourceStartedKey(user, appId, localPathStr);
|
String startedKey = getResourceStartedKey(user, appId, localPathStr);
|
||||||
String completedKey = getResourceCompletedKey(user, appId, localPathStr);
|
String completedKey = getResourceCompletedKey(user, appId, localPathStr);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Removing local resource at {}", localPathStr);
|
||||||
LOG.debug("Removing local resource at " + localPathStr);
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
WriteBatch batch = db.createWriteBatch();
|
WriteBatch batch = db.createWriteBatch();
|
||||||
try {
|
try {
|
||||||
|
@ -1505,9 +1470,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
batch.delete(key);
|
batch.delete(key);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("cleanup {} from leveldb", keyStr);
|
||||||
LOG.debug("cleanup " + keyStr + " from leveldb");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
db.write(batch);
|
db.write(batch);
|
||||||
} catch (DBException e) {
|
} catch (DBException e) {
|
||||||
|
|
|
@ -237,10 +237,8 @@ public final class DistributedScheduler extends AbstractRequestInterceptor {
|
||||||
request.setAllocatedContainers(allocatedContainers);
|
request.setAllocatedContainers(allocatedContainers);
|
||||||
request.getAllocateRequest().setAskList(partitionedAsks.getGuaranteed());
|
request.getAllocateRequest().setAskList(partitionedAsks.getGuaranteed());
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Forwarding allocate request to the" +
|
||||||
LOG.debug("Forwarding allocate request to the" +
|
|
||||||
"Distributed Scheduler Service on YARN RM");
|
"Distributed Scheduler Service on YARN RM");
|
||||||
}
|
|
||||||
|
|
||||||
DistributedSchedulingAllocateResponse dsResp =
|
DistributedSchedulingAllocateResponse dsResp =
|
||||||
getNextInterceptor().allocateForDistributedScheduling(request);
|
getNextInterceptor().allocateForDistributedScheduling(request);
|
||||||
|
|
|
@ -196,10 +196,8 @@ public class NMTokenSecretManagerInNM extends BaseNMTokenSecretManager {
|
||||||
public synchronized void appFinished(ApplicationId appId) {
|
public synchronized void appFinished(ApplicationId appId) {
|
||||||
List<ApplicationAttemptId> appAttemptList = appToAppAttemptMap.get(appId);
|
List<ApplicationAttemptId> appAttemptList = appToAppAttemptMap.get(appId);
|
||||||
if (appAttemptList != null) {
|
if (appAttemptList != null) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Removing application attempts NMToken keys for"
|
||||||
LOG.debug("Removing application attempts NMToken keys for application "
|
+ " application {}", appId);
|
||||||
+ appId);
|
|
||||||
}
|
|
||||||
for (ApplicationAttemptId appAttemptId : appAttemptList) {
|
for (ApplicationAttemptId appAttemptId : appAttemptList) {
|
||||||
removeAppAttemptKey(appAttemptId);
|
removeAppAttemptKey(appAttemptId);
|
||||||
}
|
}
|
||||||
|
@ -233,10 +231,8 @@ public class NMTokenSecretManagerInNM extends BaseNMTokenSecretManager {
|
||||||
if (oldKey == null
|
if (oldKey == null
|
||||||
|| oldKey.getMasterKey().getKeyId() != identifier.getKeyId()) {
|
|| oldKey.getMasterKey().getKeyId() != identifier.getKeyId()) {
|
||||||
// Update key only if it is modified.
|
// Update key only if it is modified.
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("NMToken key updated for application attempt : {}",
|
||||||
LOG.debug("NMToken key updated for application attempt : "
|
identifier.getApplicationAttemptId().toString());
|
||||||
+ identifier.getApplicationAttemptId().toString());
|
|
||||||
}
|
|
||||||
if (identifier.getKeyId() == currentMasterKey.getMasterKey()
|
if (identifier.getKeyId() == currentMasterKey.getMasterKey()
|
||||||
.getKeyId()) {
|
.getKeyId()) {
|
||||||
updateAppAttemptKey(appAttemptId, currentMasterKey);
|
updateAppAttemptKey(appAttemptId, currentMasterKey);
|
||||||
|
@ -252,9 +248,7 @@ public class NMTokenSecretManagerInNM extends BaseNMTokenSecretManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
public synchronized void setNodeId(NodeId nodeId) {
|
public synchronized void setNodeId(NodeId nodeId) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("updating nodeId : {}", nodeId);
|
||||||
LOG.debug("updating nodeId : " + nodeId);
|
|
||||||
}
|
|
||||||
this.nodeId = nodeId;
|
this.nodeId = nodeId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -205,18 +205,14 @@ public class NMTimelinePublisher extends CompositeService {
|
||||||
LOG.error(
|
LOG.error(
|
||||||
"Failed to publish Container metrics for container " +
|
"Failed to publish Container metrics for container " +
|
||||||
container.getContainerId());
|
container.getContainerId());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Failed to publish Container metrics for container {}",
|
||||||
LOG.debug("Failed to publish Container metrics for container " +
|
container.getContainerId(), e);
|
||||||
container.getContainerId(), e);
|
|
||||||
}
|
|
||||||
} catch (YarnException e) {
|
} catch (YarnException e) {
|
||||||
LOG.error(
|
LOG.error(
|
||||||
"Failed to publish Container metrics for container " +
|
"Failed to publish Container metrics for container " +
|
||||||
container.getContainerId(), e.getMessage());
|
container.getContainerId(), e.getMessage());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Failed to publish Container metrics for container {}",
|
||||||
LOG.debug("Failed to publish Container metrics for container " +
|
container.getContainerId(), e);
|
||||||
container.getContainerId(), e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -317,17 +313,13 @@ public class NMTimelinePublisher extends CompositeService {
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
LOG.error("Failed to publish Container metrics for container "
|
LOG.error("Failed to publish Container metrics for container "
|
||||||
+ container.getContainerId());
|
+ container.getContainerId());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Failed to publish Container metrics for container {}",
|
||||||
LOG.debug("Failed to publish Container metrics for container "
|
container.getContainerId(), e);
|
||||||
+ container.getContainerId(), e);
|
|
||||||
}
|
|
||||||
} catch (YarnException e) {
|
} catch (YarnException e) {
|
||||||
LOG.error("Failed to publish Container metrics for container "
|
LOG.error("Failed to publish Container metrics for container "
|
||||||
+ container.getContainerId(), e.getMessage());
|
+ container.getContainerId(), e.getMessage());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Failed to publish Container metrics for container {}",
|
||||||
LOG.debug("Failed to publish Container metrics for container "
|
container.getContainerId(), e);
|
||||||
+ container.getContainerId(), e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -347,8 +339,8 @@ public class NMTimelinePublisher extends CompositeService {
|
||||||
private void putEntity(TimelineEntity entity, ApplicationId appId) {
|
private void putEntity(TimelineEntity entity, ApplicationId appId) {
|
||||||
try {
|
try {
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug("Publishing the entity " + entity + ", JSON-style content: "
|
LOG.debug("Publishing the entity {} JSON-style content: {}",
|
||||||
+ TimelineUtils.dumpTimelineRecordtoJSON(entity));
|
entity, TimelineUtils.dumpTimelineRecordtoJSON(entity));
|
||||||
}
|
}
|
||||||
TimelineV2Client timelineClient = getTimelineClient(appId);
|
TimelineV2Client timelineClient = getTimelineClient(appId);
|
||||||
if (timelineClient != null) {
|
if (timelineClient != null) {
|
||||||
|
@ -359,14 +351,10 @@ public class NMTimelinePublisher extends CompositeService {
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
LOG.error("Error when publishing entity " + entity);
|
LOG.error("Error when publishing entity " + entity);
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Error when publishing entity {}", entity, e);
|
||||||
LOG.debug("Error when publishing entity " + entity, e);
|
|
||||||
}
|
|
||||||
} catch (YarnException e) {
|
} catch (YarnException e) {
|
||||||
LOG.error("Error when publishing entity " + entity, e.getMessage());
|
LOG.error("Error when publishing entity " + entity, e.getMessage());
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Error when publishing entity {}", entity, e);
|
||||||
LOG.debug("Error when publishing entity " + entity, e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -388,10 +376,8 @@ public class NMTimelinePublisher extends CompositeService {
|
||||||
break;
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{} is not a desired ApplicationEvent which"
|
||||||
LOG.debug(event.getType() + " is not a desired ApplicationEvent which"
|
+ " needs to be published by NMTimelinePublisher", event.getType());
|
||||||
+ " needs to be published by NMTimelinePublisher");
|
|
||||||
}
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -404,11 +390,8 @@ public class NMTimelinePublisher extends CompositeService {
|
||||||
break;
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{} is not a desired ContainerEvent which needs to be "
|
||||||
LOG.debug(event.getType()
|
+ " published by NMTimelinePublisher", event.getType());
|
||||||
+ " is not a desired ContainerEvent which needs to be published by"
|
|
||||||
+ " NMTimelinePublisher");
|
|
||||||
}
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -425,11 +408,8 @@ public class NMTimelinePublisher extends CompositeService {
|
||||||
ContainerMetricsConstants.LOCALIZATION_START_EVENT_TYPE);
|
ContainerMetricsConstants.LOCALIZATION_START_EVENT_TYPE);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{} is not a desired LocalizationEvent which needs to be"
|
||||||
LOG.debug(event.getType()
|
+ " published by NMTimelinePublisher", event.getType());
|
||||||
+ " is not a desired LocalizationEvent which needs to be published"
|
|
||||||
+ " by NMTimelinePublisher");
|
|
||||||
}
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -206,9 +206,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
|
||||||
throws IOException {
|
throws IOException {
|
||||||
String path = pathForCgroup(controller, groupName);
|
String path = pathForCgroup(controller, groupName);
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("createCgroup: {}", path);
|
||||||
LOG.debug("createCgroup: " + path);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!new File(path).mkdir()) {
|
if (!new File(path).mkdir()) {
|
||||||
throw new IOException("Failed to create cgroup at " + path);
|
throw new IOException("Failed to create cgroup at " + path);
|
||||||
|
@ -220,9 +218,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
|
||||||
String path = pathForCgroup(controller, groupName);
|
String path = pathForCgroup(controller, groupName);
|
||||||
param = controller + "." + param;
|
param = controller + "." + param;
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("updateCgroup: {}: {}={}", path, param, value);
|
||||||
LOG.debug("updateCgroup: " + path + ": " + param + "=" + value);
|
|
||||||
}
|
|
||||||
|
|
||||||
PrintWriter pw = null;
|
PrintWriter pw = null;
|
||||||
try {
|
try {
|
||||||
|
@ -259,7 +255,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
|
||||||
+ "/tasks"), "UTF-8"))) {
|
+ "/tasks"), "UTF-8"))) {
|
||||||
str = inl.readLine();
|
str = inl.readLine();
|
||||||
if (str != null) {
|
if (str != null) {
|
||||||
LOG.debug("First line in cgroup tasks file: " + cgf + " " + str);
|
LOG.debug("First line in cgroup tasks file: {} {}", cgf, str);
|
||||||
}
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
LOG.warn("Failed to read cgroup tasks file. ", e);
|
LOG.warn("Failed to read cgroup tasks file. ", e);
|
||||||
|
@ -302,9 +298,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
|
||||||
boolean deleteCgroup(String cgroupPath) {
|
boolean deleteCgroup(String cgroupPath) {
|
||||||
boolean deleted = false;
|
boolean deleted = false;
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("deleteCgroup: {}", cgroupPath);
|
||||||
LOG.debug("deleteCgroup: " + cgroupPath);
|
|
||||||
}
|
|
||||||
long start = clock.getTime();
|
long start = clock.getTime();
|
||||||
do {
|
do {
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -353,7 +353,7 @@ public class NodeManagerHardwareUtils {
|
||||||
for (Map.Entry<String, ResourceInformation> entry : resourceInformation
|
for (Map.Entry<String, ResourceInformation> entry : resourceInformation
|
||||||
.entrySet()) {
|
.entrySet()) {
|
||||||
ret.setResourceInformation(entry.getKey(), entry.getValue());
|
ret.setResourceInformation(entry.getKey(), entry.getValue());
|
||||||
LOG.debug("Setting key " + entry.getKey() + " to " + entry.getValue());
|
LOG.debug("Setting key {} to {}", entry.getKey(), entry.getValue());
|
||||||
}
|
}
|
||||||
if (resourceInformation.containsKey(memory)) {
|
if (resourceInformation.containsKey(memory)) {
|
||||||
Long value = resourceInformation.get(memory).getValue();
|
Long value = resourceInformation.get(memory).getValue();
|
||||||
|
@ -364,7 +364,7 @@ public class NodeManagerHardwareUtils {
|
||||||
ResourceInformation memResInfo = resourceInformation.get(memory);
|
ResourceInformation memResInfo = resourceInformation.get(memory);
|
||||||
if(memResInfo.getValue() == 0) {
|
if(memResInfo.getValue() == 0) {
|
||||||
ret.setMemorySize(getContainerMemoryMB(conf));
|
ret.setMemorySize(getContainerMemoryMB(conf));
|
||||||
LOG.debug("Set memory to " + ret.getMemorySize());
|
LOG.debug("Set memory to {}", ret.getMemorySize());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (resourceInformation.containsKey(vcores)) {
|
if (resourceInformation.containsKey(vcores)) {
|
||||||
|
@ -376,10 +376,10 @@ public class NodeManagerHardwareUtils {
|
||||||
ResourceInformation vcoresResInfo = resourceInformation.get(vcores);
|
ResourceInformation vcoresResInfo = resourceInformation.get(vcores);
|
||||||
if(vcoresResInfo.getValue() == 0) {
|
if(vcoresResInfo.getValue() == 0) {
|
||||||
ret.setVirtualCores(getVCores(conf));
|
ret.setVirtualCores(getVCores(conf));
|
||||||
LOG.debug("Set vcores to " + ret.getVirtualCores());
|
LOG.debug("Set vcores to {}", ret.getVirtualCores());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
LOG.debug("Node resource information map is " + ret);
|
LOG.debug("Node resource information map is {}", ret);
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -49,9 +49,7 @@ public class ProcessIdFileReader {
|
||||||
if (path == null) {
|
if (path == null) {
|
||||||
throw new IOException("Trying to access process id from a null path");
|
throw new IOException("Trying to access process id from a null path");
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Accessing pid from pid file {}", path);
|
||||||
LOG.debug("Accessing pid from pid file " + path);
|
|
||||||
}
|
|
||||||
String processId = null;
|
String processId = null;
|
||||||
BufferedReader bufReader = null;
|
BufferedReader bufReader = null;
|
||||||
|
|
||||||
|
@ -99,10 +97,8 @@ public class ProcessIdFileReader {
|
||||||
bufReader.close();
|
bufReader.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Got pid {} from path {}",
|
||||||
LOG.debug("Got pid " + (processId != null ? processId : "null")
|
(processId != null ? processId : "null"), path);
|
||||||
+ " from path " + path);
|
|
||||||
}
|
|
||||||
return processId;
|
return processId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -151,9 +151,7 @@ public class ContainerLogsPage extends NMView {
|
||||||
printAggregatedLogFileDirectory(html, containersLogMeta);
|
printAggregatedLogFileDirectory(html, containersLogMeta);
|
||||||
}
|
}
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{}", ex);
|
||||||
LOG.debug(ex.getMessage());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -328,9 +328,7 @@ public class NMWebServices {
|
||||||
} catch (IOException ex) {
|
} catch (IOException ex) {
|
||||||
// Something wrong with we tries to access the remote fs for the logs.
|
// Something wrong with we tries to access the remote fs for the logs.
|
||||||
// Skip it and do nothing
|
// Skip it and do nothing
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("{}", ex);
|
||||||
LOG.debug(ex.getMessage());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
GenericEntity<List<ContainerLogsInfo>> meta = new GenericEntity<List<
|
GenericEntity<List<ContainerLogsInfo>> meta = new GenericEntity<List<
|
||||||
ContainerLogsInfo>>(containersLogsInfo){};
|
ContainerLogsInfo>>(containersLogsInfo){};
|
||||||
|
@ -433,10 +431,8 @@ public class NMWebServices {
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
// This NM does not have this container any more. We
|
// This NM does not have this container any more. We
|
||||||
// assume the container has already finished.
|
// assume the container has already finished.
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Can not find the container:{} in this node.",
|
||||||
LOG.debug("Can not find the container:" + containerId
|
containerId);
|
||||||
+ " in this node.");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
final boolean isRunning = tempIsRunning;
|
final boolean isRunning = tempIsRunning;
|
||||||
File logFile = null;
|
File logFile = null;
|
||||||
|
|
|
@ -213,10 +213,8 @@ public class ActiveStandbyElectorBasedElectorService extends AbstractService
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void fenceOldActive(byte[] oldActiveData) {
|
public void fenceOldActive(byte[] oldActiveData) {
|
||||||
if (LOG.isDebugEnabled()) {
|
LOG.debug("Request to fence old active being ignored, " +
|
||||||
LOG.debug("Request to fence old active being ignored, " +
|
"as embedded leader election doesn't support fencing");
|
||||||
"as embedded leader election doesn't support fencing");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static byte[] createActiveNodeInfo(String clusterId, String rmId)
|
private static byte[] createActiveNodeInfo(String clusterId, String rmId)
|
||||||
|
|
|
@ -1470,10 +1470,8 @@ public class ClientRMService extends AbstractService implements
|
||||||
ReservationDefinition contract, String reservationId) {
|
ReservationDefinition contract, String reservationId) {
|
||||||
if ((contract.getArrival() - clock.getTime()) < reservationSystem
|
if ((contract.getArrival() - clock.getTime()) < reservationSystem
|
||||||
.getPlanFollowerTimeStep()) {
|
.getPlanFollowerTimeStep()) {
|
||||||
LOG.debug(MessageFormat
|
LOG.debug("Reservation {} is within threshold so attempting to"
|
||||||
.format(
|
+ " create synchronously.", reservationId);
|
||||||
"Reservation {0} is within threshold so attempting to create synchronously.",
|
|
||||||
reservationId));
|
|
||||||
reservationSystem.synchronizePlan(planName, true);
|
reservationSystem.synchronizePlan(planName, true);
|
||||||
LOG.info(MessageFormat.format("Created reservation {0} synchronously.",
|
LOG.info(MessageFormat.format("Created reservation {0} synchronously.",
|
||||||
reservationId));
|
reservationId));
|
||||||
|
|
|
@ -292,7 +292,7 @@ public class DecommissioningNodesWatcher {
|
||||||
}
|
}
|
||||||
// Remove stale non-DECOMMISSIONING node
|
// Remove stale non-DECOMMISSIONING node
|
||||||
if (d.nodeState != NodeState.DECOMMISSIONING) {
|
if (d.nodeState != NodeState.DECOMMISSIONING) {
|
||||||
LOG.debug("remove " + d.nodeState + " " + d.nodeId);
|
LOG.debug("remove {} {}", d.nodeState, d.nodeId);
|
||||||
it.remove();
|
it.remove();
|
||||||
continue;
|
continue;
|
||||||
} else if (now - d.lastUpdateTime > 60000L) {
|
} else if (now - d.lastUpdateTime > 60000L) {
|
||||||
|
@ -300,7 +300,7 @@ public class DecommissioningNodesWatcher {
|
||||||
RMNode rmNode = getRmNode(d.nodeId);
|
RMNode rmNode = getRmNode(d.nodeId);
|
||||||
if (rmNode != null &&
|
if (rmNode != null &&
|
||||||
rmNode.getState() == NodeState.DECOMMISSIONED) {
|
rmNode.getState() == NodeState.DECOMMISSIONED) {
|
||||||
LOG.debug("remove " + rmNode.getState() + " " + d.nodeId);
|
LOG.debug("remove {} {}", rmNode.getState(), d.nodeId);
|
||||||
it.remove();
|
it.remove();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
@ -308,7 +308,7 @@ public class DecommissioningNodesWatcher {
|
||||||
if (d.timeoutMs >= 0 &&
|
if (d.timeoutMs >= 0 &&
|
||||||
d.decommissioningStartTime + d.timeoutMs < now) {
|
d.decommissioningStartTime + d.timeoutMs < now) {
|
||||||
staleNodes.add(d.nodeId);
|
staleNodes.add(d.nodeId);
|
||||||
LOG.debug("Identified stale and timeout node " + d.nodeId);
|
LOG.debug("Identified stale and timeout node {}", d.nodeId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -342,14 +342,14 @@ public class DecommissioningNodesWatcher {
|
||||||
ApplicationId appId = it.next();
|
ApplicationId appId = it.next();
|
||||||
RMApp rmApp = rmContext.getRMApps().get(appId);
|
RMApp rmApp = rmContext.getRMApps().get(appId);
|
||||||
if (rmApp == null) {
|
if (rmApp == null) {
|
||||||
LOG.debug("Consider non-existing app " + appId + " as completed");
|
LOG.debug("Consider non-existing app {} as completed", appId);
|
||||||
it.remove();
|
it.remove();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (rmApp.getState() == RMAppState.FINISHED ||
|
if (rmApp.getState() == RMAppState.FINISHED ||
|
||||||
rmApp.getState() == RMAppState.FAILED ||
|
rmApp.getState() == RMAppState.FAILED ||
|
||||||
rmApp.getState() == RMAppState.KILLED) {
|
rmApp.getState() == RMAppState.KILLED) {
|
||||||
LOG.debug("Remove " + rmApp.getState() + " app " + appId);
|
LOG.debug("Remove {} app {}", rmApp.getState(), appId);
|
||||||
it.remove();
|
it.remove();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -493,17 +493,17 @@ public class NodesListManager extends CompositeService implements
|
||||||
RMNode eventNode = event.getNode();
|
RMNode eventNode = event.getNode();
|
||||||
switch (event.getType()) {
|
switch (event.getType()) {
|
||||||
case NODE_UNUSABLE:
|
case NODE_UNUSABLE:
|
||||||
LOG.debug(eventNode + " reported unusable");
|
LOG.debug("{} reported unusable", eventNode);
|
||||||
sendRMAppNodeUpdateEventToNonFinalizedApps(eventNode,
|
sendRMAppNodeUpdateEventToNonFinalizedApps(eventNode,
|
||||||
RMAppNodeUpdateType.NODE_UNUSABLE);
|
RMAppNodeUpdateType.NODE_UNUSABLE);
|
||||||
break;
|
break;
|
||||||
case NODE_USABLE:
|
case NODE_USABLE:
|
||||||
LOG.debug(eventNode + " reported usable");
|
LOG.debug("{} reported usable", eventNode);
|
||||||
sendRMAppNodeUpdateEventToNonFinalizedApps(eventNode,
|
sendRMAppNodeUpdateEventToNonFinalizedApps(eventNode,
|
||||||
RMAppNodeUpdateType.NODE_USABLE);
|
RMAppNodeUpdateType.NODE_USABLE);
|
||||||
break;
|
break;
|
||||||
case NODE_DECOMMISSIONING:
|
case NODE_DECOMMISSIONING:
|
||||||
LOG.debug(eventNode + " reported decommissioning");
|
LOG.debug("{} reported decommissioning", eventNode);
|
||||||
sendRMAppNodeUpdateEventToNonFinalizedApps(
|
sendRMAppNodeUpdateEventToNonFinalizedApps(
|
||||||
eventNode, RMAppNodeUpdateType.NODE_DECOMMISSIONING);
|
eventNode, RMAppNodeUpdateType.NODE_DECOMMISSIONING);
|
||||||
break;
|
break;
|
||||||
|
|
|
@ -618,8 +618,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
|
||||||
@Override
|
@Override
|
||||||
public void handle(RMAppManagerEvent event) {
|
public void handle(RMAppManagerEvent event) {
|
||||||
ApplicationId applicationId = event.getApplicationId();
|
ApplicationId applicationId = event.getApplicationId();
|
||||||
LOG.debug("RMAppManager processing event for "
|
LOG.debug("RMAppManager processing event for {} of type {}",
|
||||||
+ applicationId + " of type " + event.getType());
|
applicationId, event.getType());
|
||||||
switch (event.getType()) {
|
switch (event.getType()) {
|
||||||
case APP_COMPLETED :
|
case APP_COMPLETED :
|
||||||
finishApplication(applicationId);
|
finishApplication(applicationId);
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue