YARN-5993. Allow native services quicklinks to be exported for each component. Contributed by Billie Rinaldi

This commit is contained in:
Gour Saha 2016-12-28 15:24:36 -08:00 committed by Jian He
parent 7d685f2fb3
commit a5e20f0fc1
6 changed files with 178 additions and 174 deletions

View File

@ -183,17 +183,17 @@ public final class SliderUtils {
return !isUnset(s);
}
public static boolean isEmpty(List l) {
public static boolean isEmpty(Collection l) {
return l == null || l.isEmpty();
}
/**
* Probe for a list existing and not being empty
* @param l list
* Probe for a collection existing and not being empty
* @param l collection
* @return true if the reference is valid and it contains entries
*/
public static boolean isNotEmpty(List l) {
public static boolean isNotEmpty(Collection l) {
return l != null && !l.isEmpty();
}

View File

@ -106,6 +106,26 @@ public class ExportEntry {
this.validUntil = validUntil;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ExportEntry that = (ExportEntry) o;
if (value != null ? !value.equals(that.value) : that.value != null)
return false;
return containerId != null ? containerId.equals(that.containerId) :
that.containerId == null;
}
@Override
public int hashCode() {
int result = value != null ? value.hashCode() : 0;
result = 31 * result + (containerId != null ? containerId.hashCode() : 0);
return result;
}
@Override
public String toString() {
return new StringBuilder("ExportEntry{").

View File

@ -26,8 +26,10 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.io.IOException;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
/**
* JSON-serializable description of a published key-val configuration.
@ -41,7 +43,7 @@ public class PublishedExports {
public String description;
public long updated;
public String updatedTime;
public Map<String, List<ExportEntry>> entries = new HashMap<>();
public Map<String, Set<ExportEntry>> entries = new HashMap<>();
public PublishedExports() {
}
@ -62,7 +64,7 @@ public class PublishedExports {
* @param entries entries to put
*/
public PublishedExports(String description,
Iterable<Map.Entry<String, List<ExportEntry>>> entries) {
Iterable<Entry<String, Set<ExportEntry>>> entries) {
this.description = description;
putValues(entries);
}
@ -87,15 +89,22 @@ public class PublishedExports {
this.updatedTime = new Date(updated).toString();
}
public Map<String, Set<ExportEntry>> sortedEntries() {
Map<String, Set<ExportEntry>> sortedEntries = new TreeMap<>();
sortedEntries.putAll(entries);
return sortedEntries;
}
/**
* Set the values from an iterable (this includes a Hadoop Configuration and Java properties
* object). Any existing value set is discarded
*
* @param entries entries to put
* @param values values to put
*/
public void putValues(Iterable<Map.Entry<String, List<ExportEntry>>> entries) {
this.entries = new HashMap<String, List<ExportEntry>>();
for (Map.Entry<String, List<ExportEntry>> entry : entries) {
public void putValues(Iterable<Map.Entry<String, Set<ExportEntry>>> values) {
this.entries = new HashMap<>();
for (Map.Entry<String, Set<ExportEntry>> entry : values) {
this.entries.put(entry.getKey(), entry.getValue());
}
}

View File

@ -75,6 +75,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Pattern;
/**
@ -1053,32 +1054,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
return hosts;
}
/**
* Return a list of hostnames based on current ClusterNodes.
* @param values cluster nodes
* @return list of hosts
*/
public Iterable<String> getHostNamesList(Collection<ClusterNode> values) {
List<String> hosts = new ArrayList<>();
for (ClusterNode cn : values) {
hosts.add(cn.hostname);
}
return hosts;
}
/**
* Return a list of IPs based on current ClusterNodes.
* @param values cluster nodes
* @return list of hosts
*/
public Iterable<String> getIPsList(Collection<ClusterNode> values) {
List<String> hosts = new ArrayList<>();
for (ClusterNode cn : values) {
hosts.add(cn.ip);
}
return hosts;
}
/**
* Update ServiceRecord in Registry with IP and hostname.
* @param amState access to AM state
@ -1148,27 +1123,30 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
* Publish an export group.
* @param exportGroup export groups
* @param amState access to AM state
* @param roleGroup component group
* @param groupName export group name
*/
public void publishExportGroup(Map<String, List<ExportEntry>> exportGroup,
StateAccessForProviders amState, String roleGroup) {
public void publishExportGroup(
Map<String, Set<ExportEntry>> exportGroup,
StateAccessForProviders amState, String groupName) {
// Publish in old format for the time being
Map<String, String> simpleEntries = new HashMap<>();
for (Entry<String, List<ExportEntry>> entry : exportGroup.entrySet()) {
List<ExportEntry> exports = entry.getValue();
for (Entry<String, Set<ExportEntry>> entry : exportGroup.entrySet()) {
Set<ExportEntry> exports = entry.getValue();
if (SliderUtils.isNotEmpty(exports)) {
// there is no support for multiple exports per name, so extract only
// the first one
simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue());
Set<String> values = new TreeSet<>();
for (ExportEntry export : exports) {
values.add(export.getValue());
}
simpleEntries.put(entry.getKey(), StringUtils.join(",", values));
}
}
publishApplicationInstanceData(roleGroup, roleGroup,
publishApplicationInstanceData(groupName, groupName,
simpleEntries.entrySet(), amState);
PublishedExports exports = new PublishedExports(roleGroup);
PublishedExports exports = new PublishedExports(groupName);
exports.setUpdated(new Date().getTime());
exports.putValues(exportGroup.entrySet());
amState.getPublishedExportsSet().put(roleGroup, exports);
amState.getPublishedExportsSet().put(groupName, exports);
}
public Map<String, String> getExports(ConfTreeOperations appConf,
@ -1179,75 +1157,26 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
return exports;
}
private static final String COMPONENT_TAG = "component";
private static final String HOST_FOLDER_FORMAT = "%s:%s";
private static final String CONTAINER_LOGS_TAG = "container_log_dirs";
private static final String CONTAINER_PWDS_TAG = "container_work_dirs";
/**
* Format the folder locations and publish in the registry service.
* @param folders folder information
* @param containerId container ID
* @param hostFqdn host FQDN
* @param componentName component name
*/
public void publishFolderPaths(Map<String, String> folders,
String containerId, String componentName, String hostFqdn,
StateAccessForProviders amState,
Map<String, ExportEntry> logFolderExports,
Map<String, ExportEntry> workFolderExports) {
Date now = new Date();
for (Map.Entry<String, String> entry : folders.entrySet()) {
ExportEntry exportEntry = new ExportEntry();
exportEntry.setValue(String.format(HOST_FOLDER_FORMAT, hostFqdn,
entry.getValue()));
exportEntry.setContainerId(containerId);
exportEntry.setLevel(COMPONENT_TAG);
exportEntry.setTag(componentName);
exportEntry.setUpdatedTime(now.toString());
if (entry.getKey().equals("AGENT_LOG_ROOT") ||
entry.getKey().equals("LOG_DIR")) {
synchronized (logFolderExports) {
logFolderExports.put(containerId, exportEntry);
}
} else {
synchronized (workFolderExports) {
workFolderExports.put(containerId, exportEntry);
}
}
log.info("Updating log and pwd folders for container {}", containerId);
public String getGroupKey(String roleGroup, ConfTreeOperations appConf) {
String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
return getNameOrGroupKey(rolePrefix, roleGroup);
}
PublishedExports exports = new PublishedExports(CONTAINER_LOGS_TAG);
exports.setUpdated(now.getTime());
synchronized (logFolderExports) {
updateExportsFromList(exports, logFolderExports);
}
amState.getPublishedExportsSet().put(CONTAINER_LOGS_TAG, exports);
exports = new PublishedExports(CONTAINER_PWDS_TAG);
exports.setUpdated(now.getTime());
synchronized (workFolderExports) {
updateExportsFromList(exports, workFolderExports);
}
amState.getPublishedExportsSet().put(CONTAINER_PWDS_TAG, exports);
public String getNameKey(String roleName, String roleGroup,
ConfTreeOperations appConf) {
String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
return getNameOrGroupKey(rolePrefix, roleName);
}
/**
* Update the export data from the map.
* @param exports published exports
* @param folderExports folder exports
*/
private void updateExportsFromList(PublishedExports exports,
Map<String, ExportEntry> folderExports) {
Map<String, List<ExportEntry>> perComponentList = new HashMap<>();
for(Map.Entry<String, ExportEntry> logEntry : folderExports.entrySet()) {
String componentName = logEntry.getValue().getTag();
if (!perComponentList.containsKey(componentName)) {
perComponentList.put(componentName, new ArrayList<ExportEntry>());
public String getNameOrGroupKey(String rolePrefix, String roleNameOrGroup) {
if (!rolePrefix.isEmpty()) {
if (!roleNameOrGroup.startsWith(rolePrefix)) {
log.warn("Something went wrong, {} doesn't start with {}",
roleNameOrGroup, rolePrefix);
return null;
}
perComponentList.get(componentName).add(logEntry.getValue());
roleNameOrGroup = roleNameOrGroup.substring(rolePrefix.length());
}
exports.putValues(perComponentList.entrySet());
return roleNameOrGroup.toUpperCase(Locale.ENGLISH);
}
}

View File

@ -46,27 +46,24 @@ import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Scanner;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
public class DockerProviderService extends AbstractProviderService implements
ProviderCore,
DockerKeys,
@ -77,10 +74,16 @@ public class DockerProviderService extends AbstractProviderService implements
private static final ProviderUtils providerUtils = new ProviderUtils(log);
private static final String EXPORT_GROUP = "quicklinks";
private static final String APPLICATION_TAG = "application";
private static final String HOST_KEY_FORMAT = "${%s_HOST}";
private static final String IP_KEY_FORMAT = "${%s_IP}";
private static final String VARIABLE_INDICATOR = "${";
private String clusterName = null;
private SliderFileSystem fileSystem = null;
private final Map<String, Set<ExportEntry>> exportMap =
new ConcurrentHashMap<>();
protected DockerProviderService() {
super("DockerProviderService");
}
@ -118,9 +121,6 @@ public class DockerProviderService extends AbstractProviderService implements
String roleName = providerRole.name;
String roleGroup = providerRole.group;
initializeApplicationConfiguration(instanceDefinition, fileSystem,
roleGroup);
log.info("Build launch context for Docker");
log.debug(instanceDefinition.toString());
@ -279,6 +279,23 @@ public class DockerProviderService extends AbstractProviderService implements
}
}
@Override
public void notifyContainerCompleted(ContainerId containerId) {
if (containerId != null) {
String containerIdStr = containerId.toString();
log.info("Removing container exports for {}", containerIdStr);
for (Set<ExportEntry> exportEntries : exportMap.values()) {
for (Iterator<ExportEntry> iter = exportEntries.iterator();
iter.hasNext();) {
ExportEntry entry = iter.next();
if (containerIdStr.equals(entry.getContainerId())) {
iter.remove();
}
}
}
}
}
@Override
public boolean processContainerStatus(ContainerId containerId,
ContainerStatus status) {
@ -301,7 +318,7 @@ public class DockerProviderService extends AbstractProviderService implements
containerIdStr, roleName, status.getIPs(), status.getHost());
publishExportGroups(containerIdStr, roleName, roleGroup,
status.getHost());
status.getHost(), status.getIPs());
return false;
}
@ -312,17 +329,13 @@ public class DockerProviderService extends AbstractProviderService implements
* are substituted with the actual hostnames of the containers.
*/
protected void publishExportGroups(String containerId,
String roleName, String roleGroup, String thisHost) {
String roleName, String roleGroup, String thisHost, List<String> ips) {
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
Map<String, String> exports = providerUtils.getExports(
getAmState().getAppConfSnapshot(), roleGroup);
String hostKeyFormat = "${%s_HOST}";
String hostNameKeyFormat = "${%s_HOSTNAME}";
String ipKeyFormat = "${%s_IP}";
// publish export groups if any
Map<String, String> standardTokens = providerUtils.getStandardTokenMap(
appConf, internalsConf, roleName, roleGroup, containerId,
@ -331,38 +344,33 @@ public class DockerProviderService extends AbstractProviderService implements
appConf.getComponent(roleGroup).options, standardTokens);
replaceTokens.putAll(standardTokens);
String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
for (Map.Entry<String, Map<String, ClusterNode>> entry :
getAmState().getRoleClusterNodeMapping().entrySet()) {
String otherRolePrefix = appConf.getComponentOpt(entry.getKey(),
ROLE_PREFIX, "");
if (!otherRolePrefix.equals(rolePrefix)) {
// hostname replacements are only made within role prefix groups
continue;
String roleNameKey = providerUtils.getNameKey(roleName, roleGroup,
appConf);
String roleNameIPKey = null;
if (roleNameKey != null) {
replaceTokens.put(String.format(HOST_KEY_FORMAT, roleNameKey), thisHost);
roleNameIPKey = Pattern.quote(String.format(IP_KEY_FORMAT, roleNameKey));
} else {
// should not happen, but log if it does
log.info("Not replacing HOST or IP tokens because key was null for {}",
roleName);
}
String key = entry.getKey();
if (!rolePrefix.isEmpty()) {
if (!key.startsWith(rolePrefix)) {
log.warn("Something went wrong, {} doesn't start with {}", key,
rolePrefix);
continue;
String roleGroupKey = providerUtils.getGroupKey(roleGroup, appConf);
String roleGroupIPKey = null;
if (roleGroupKey != null) {
if (roleNameKey == null || !roleGroupKey.equals(roleNameKey)) {
replaceTokens.put(String.format(HOST_KEY_FORMAT, roleGroupKey),
thisHost);
roleGroupIPKey = Pattern.quote(String.format(IP_KEY_FORMAT,
roleGroupKey));
}
key = key.substring(rolePrefix.length());
}
key = key.toUpperCase(Locale.ENGLISH);
String host = providerUtils.getHostsList(
entry.getValue().values(), true).iterator().next();
replaceTokens.put(String.format(hostKeyFormat, key), host);
String hostName = providerUtils.getHostNamesList(
entry.getValue().values()).iterator().next();
replaceTokens.put(String.format(hostNameKeyFormat, key), hostName);
String ip = providerUtils.getIPsList(
entry.getValue().values()).iterator().next();
replaceTokens.put(String.format(ipKeyFormat, key), ip);
} else {
// should not happen, but log if it does
log.info("Not replacing HOST or IP tokens because key was null for {}",
roleGroup);
}
replaceTokens.put("${THIS_HOST}", thisHost);
Map<String, List<ExportEntry>> entries = new HashMap<>();
for (Entry<String, String> export : exports.entrySet()) {
String value = export.getValue();
// replace host names and site properties
@ -372,18 +380,53 @@ public class DockerProviderService extends AbstractProviderService implements
value = value.replaceAll(Pattern.quote(token), entry.getValue());
}
}
Set<String> values = new HashSet<>();
for (String ip : ips) {
values.add(substituteIP(roleNameIPKey, roleGroupIPKey, ip, value));
}
for (String exportValue : values) {
if (exportValue.contains(VARIABLE_INDICATOR)) {
// not all variables have been substituted, so do not export
continue;
}
ExportEntry entry = new ExportEntry();
entry.setContainerId(containerId);
entry.setLevel(APPLICATION_TAG);
entry.setValue(value);
entry.setValue(exportValue);
entry.setUpdatedTime(new Date().toString());
// over-write, app exports are singletons
entries.put(export.getKey(), new ArrayList(Arrays.asList(entry)));
log.info("Preparing to publish. Key {} and Value {}",
export.getKey(), value);
Set<ExportEntry> exportEntries = getExportEntries(export.getKey());
exportEntries.add(entry);
log.info("Preparing to publish for {}. Key {} and Value {}",
roleName, export.getKey(), entry);
}
if (!entries.isEmpty()) {
providerUtils.publishExportGroup(entries, getAmState(), EXPORT_GROUP);
}
if (!exportMap.isEmpty()) {
providerUtils.publishExportGroup(exportMap, getAmState(), EXPORT_GROUP);
}
}
protected String substituteIP(String roleNameIPKey, String roleGroupIPKey,
String ip, String value) {
if (roleNameIPKey != null) {
value = value.replaceAll(roleNameIPKey, ip);
}
if (roleGroupIPKey != null) {
value = value.replaceAll(roleGroupIPKey, ip);
}
return value;
}
protected Set<ExportEntry> getExportEntries(String key) {
if (!this.exportMap.containsKey(key)) {
synchronized (this.exportMap) {
if (!this.exportMap.containsKey(key)) {
this.exportMap.put(key, Collections.newSetFromMap(
new ConcurrentHashMap<>()));
}
}
}
return this.exportMap.get(key);
}
@Override

View File

@ -25,7 +25,6 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.registry.docstore.ExportEntry;
import org.apache.slider.core.registry.docstore.PublishedExports;
@ -42,6 +41,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS;
@ -291,13 +291,16 @@ public class IndexBlock extends SliderHamletBlock {
LI<UL<Hamlet>> item = ul.li();
item.span().$class("bold")._(export.description)._();
UL sublist = item.ul();
for (Entry<String, List<ExportEntry>> entry : export.entries.entrySet()) {
for (Entry<String, Set<ExportEntry>> entry : export.sortedEntries()
.entrySet()) {
if (SliderUtils.isNotEmpty(entry.getValue())) {
LI sublistItem = sublist.li()._(entry.getKey());
for (ExportEntry exportEntry : entry.getValue()) {
sublistItem._(exportEntry.getValue());
}
sublistItem._();
}
}
sublist._();
item._();
}