diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index fe4d3ac05b0..93764e67e84 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -90,6 +90,9 @@ Release 2.5.0 - UNRELEASED YARN-2017. Merged some of the common scheduler code. (Jian He via vinodkv) + YARN-2049. Added delegation-token support for the Timeline Server. (Zhijie + Shen via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java new file mode 100644 index 00000000000..90d13a2b0c4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineDelegationTokenResponse.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.timeline; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * The response of delegation token related request + */ +@XmlRootElement(name = "delegationtoken") +@XmlAccessorType(XmlAccessType.NONE) +@Public +@Unstable +public class TimelineDelegationTokenResponse { + + private String type; + private Object content; + + public TimelineDelegationTokenResponse() { + + } + + @XmlElement(name = "type") + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + @XmlElement(name = "content") + public Object getContent() { + return content; + } + + public void setContent(Object content) { + this.content = content; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index e25a941f970..48763b3120e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1136,14 +1136,6 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT; - /**The kerberos principal to be used for spnego filter for timeline service.*/ - public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY = - TIMELINE_SERVICE_PREFIX + "webapp.spnego-principal"; - - /**The kerberos keytab to be used for spnego filter for timeline service.*/ - public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = - TIMELINE_SERVICE_PREFIX + "webapp.spnego-keytab-file"; - /** Timeline service store class */ public static final String TIMELINE_SERVICE_STORE = TIMELINE_SERVICE_PREFIX + "store-class"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java index fda0a4f18e6..a2ed3e70a51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java @@ -32,8 +32,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException; /** * A client library that can be used to post some information in terms of a * number of conceptual entities. - * - * @See Entity */ @Public @Unstable diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java new file mode 100644 index 00000000000..f4f1507a4f2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java @@ -0,0 +1,252 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api.impl; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.client.Authenticator; +import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenOperation; +import org.apache.hadoop.yarn.security.client.TimelineAuthenticationConsts; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * A KerberosAuthenticator subclass that fallback to + * {@link TimelineAuthenticationConsts}. + */ +@Private +@Unstable +public class TimelineAuthenticator extends KerberosAuthenticator { + + private static ObjectMapper mapper; + + static { + mapper = new ObjectMapper(); + YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); + } + + /** + * Returns the fallback authenticator if the server does not use Kerberos + * SPNEGO HTTP authentication. + * + * @return a {@link TimelineAuthenticationConsts} instance. + */ + @Override + protected Authenticator getFallBackAuthenticator() { + return new TimelineAuthenticator(); + } + + public static void injectDelegationToken(Map params, + Token dtToken) + throws IOException { + if (dtToken != null) { + params.put(TimelineAuthenticationConsts.DELEGATION_PARAM, + dtToken.encodeToUrlString()); + } + } + + private boolean hasDelegationToken(URL url) { + return url.getQuery().contains( + TimelineAuthenticationConsts.DELEGATION_PARAM + "="); + } + + @Override + public void authenticate(URL url, AuthenticatedURL.Token token) + throws IOException, AuthenticationException { + if (!hasDelegationToken(url)) { + super.authenticate(url, token); + } + } + + public static Token getDelegationToken( + URL url, AuthenticatedURL.Token token, String renewer) throws IOException { + TimelineDelegationTokenOperation op = + TimelineDelegationTokenOperation.GETDELEGATIONTOKEN; + Map params = new HashMap(); + params.put(TimelineAuthenticationConsts.OP_PARAM, op.toString()); + params.put(TimelineAuthenticationConsts.RENEWER_PARAM, renewer); + url = appendParams(url, params); + AuthenticatedURL aUrl = + new AuthenticatedURL(new TimelineAuthenticator()); + try { + HttpURLConnection conn = aUrl.openConnection(url, token); + conn.setRequestMethod(op.getHttpMethod()); + TimelineDelegationTokenResponse dtRes = validateAndParseResponse(conn); + if (!dtRes.getType().equals( + TimelineAuthenticationConsts.DELEGATION_TOKEN_URL)) { + throw new IOException("The response content is not expected: " + + dtRes.getContent()); + } + String tokenStr = dtRes.getContent().toString(); + Token dToken = + new Token(); + dToken.decodeFromUrlString(tokenStr); + return dToken; + } catch (AuthenticationException ex) { + throw new IOException(ex.toString(), ex); + } + } + + public static long renewDelegationToken(URL url, + AuthenticatedURL.Token token, + Token dToken) throws IOException { + Map params = new HashMap(); + params.put(TimelineAuthenticationConsts.OP_PARAM, + TimelineDelegationTokenOperation.RENEWDELEGATIONTOKEN.toString()); + params.put(TimelineAuthenticationConsts.TOKEN_PARAM, + dToken.encodeToUrlString()); + url = appendParams(url, params); + AuthenticatedURL aUrl = + new AuthenticatedURL(new TimelineAuthenticator()); + try { + HttpURLConnection conn = aUrl.openConnection(url, token); + conn.setRequestMethod( + TimelineDelegationTokenOperation.RENEWDELEGATIONTOKEN.getHttpMethod()); + TimelineDelegationTokenResponse dtRes = validateAndParseResponse(conn); + if (!dtRes.getType().equals( + TimelineAuthenticationConsts.DELEGATION_TOKEN_EXPIRATION_TIME)) { + throw new IOException("The response content is not expected: " + + dtRes.getContent()); + } + return Long.valueOf(dtRes.getContent().toString()); + } catch (AuthenticationException ex) { + throw new IOException(ex.toString(), ex); + } + } + + public static void cancelDelegationToken(URL url, + AuthenticatedURL.Token token, + Token dToken) throws IOException { + Map params = new HashMap(); + params.put(TimelineAuthenticationConsts.OP_PARAM, + TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN.toString()); + params.put(TimelineAuthenticationConsts.TOKEN_PARAM, + dToken.encodeToUrlString()); + url = appendParams(url, params); + AuthenticatedURL aUrl = + new AuthenticatedURL(new TimelineAuthenticator()); + try { + HttpURLConnection conn = aUrl.openConnection(url, token); + conn.setRequestMethod(TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN + .getHttpMethod()); + validateAndParseResponse(conn); + } catch (AuthenticationException ex) { + throw new IOException(ex.toString(), ex); + } + } + + /** + * Convenience method that appends parameters an HTTP URL. + * + * @param url + * the url. + * @param params + * the query string parameters. + * + * @return a URL + * + * @throws IOException + * thrown if an IO error occurs. + */ + public static URL appendParams(URL url, Map params) + throws IOException { + StringBuilder sb = new StringBuilder(); + sb.append(url); + String separator = url.toString().contains("?") ? "&" : "?"; + for (Map.Entry entry : params.entrySet()) { + sb.append(separator).append(entry.getKey()).append("="). + append(URLEncoder.encode(entry.getValue(), "UTF8")); + separator = "&"; + } + return new URL(sb.toString()); + } + + /** + * Validates the response of an HttpURLConnection. If the current + * status code is not 200, it will throw an exception with a detail message + * using Server side error messages if available. Otherwise, + * {@link TimelineDelegationTokenResponse} will be parsed and returned. + * + * @param conn + * the HttpURLConnection. + * @return + * @throws IOException + * thrown if the current status code is not 200 or the JSON response + * cannot be parsed correctly + */ + private static TimelineDelegationTokenResponse validateAndParseResponse( + HttpURLConnection conn) throws IOException { + int status = conn.getResponseCode(); + JsonNode json = mapper.readTree(conn.getInputStream()); + if (status == HttpURLConnection.HTTP_OK) { + return mapper.readValue(json, TimelineDelegationTokenResponse.class); + } else { + // If the status code is not 200, some thing wrong should happen at the + // server side, the JSON content is going to contain exception details. + // We can use the JSON content to reconstruct the exception object. + try { + String message = + json.get(TimelineAuthenticationConsts.ERROR_MESSAGE_JSON) + .getTextValue(); + String exception = + json.get(TimelineAuthenticationConsts.ERROR_EXCEPTION_JSON) + .getTextValue(); + String className = + json.get(TimelineAuthenticationConsts.ERROR_CLASSNAME_JSON) + .getTextValue(); + + try { + ClassLoader cl = TimelineAuthenticator.class.getClassLoader(); + Class klass = cl.loadClass(className); + Constructor constr = klass.getConstructor(String.class); + throw (IOException) constr.newInstance(message); + } catch (IOException ex) { + throw ex; + } catch (Exception ex) { + throw new IOException(MessageFormat.format("{0} - {1}", exception, + message)); + } + } catch (IOException ex) { + if (ex.getCause() instanceof IOException) { + throw (IOException) ex.getCause(); + } + throw new IOException( + MessageFormat.format("HTTP status [{0}], {1}", + status, conn.getResponseMessage())); + } + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java new file mode 100644 index 00000000000..d2b135851a5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTimelineSecurityInfo.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import java.lang.annotation.Annotation; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.KerberosInfo; +import org.apache.hadoop.security.SecurityInfo; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenInfo; +import org.apache.hadoop.security.token.TokenSelector; +import org.apache.hadoop.yarn.api.ApplicationHistoryProtocolPB; +import org.apache.hadoop.yarn.conf.YarnConfiguration; + + +@Public +@Unstable +public class ClientTimelineSecurityInfo extends SecurityInfo { + @Override + public KerberosInfo getKerberosInfo(Class protocol, Configuration conf) { + if (!protocol + .equals(ApplicationHistoryProtocolPB.class)) { + return null; + } + return new KerberosInfo() { + + @Override + public Class annotationType() { + return null; + } + + @Override + public String serverPrincipal() { + return YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL; + } + + @Override + public String clientPrincipal() { + return null; + } + }; + } + + @Override + public TokenInfo getTokenInfo(Class protocol, Configuration conf) { + if (!protocol + .equals(ApplicationHistoryProtocolPB.class)) { + return null; + } + return new TokenInfo() { + + @Override + public Class annotationType() { + return null; + } + + @Override + public Class> + value() { + return TimelineDelegationTokenSelector.class; + } + }; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineAuthenticationConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineAuthenticationConsts.java new file mode 100644 index 00000000000..e89e1a66aa1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineAuthenticationConsts.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * The constants that are going to be used by the timeline Kerberos + delegation + * token authentication. + */ + +@Private +@Unstable +public class TimelineAuthenticationConsts { + + public static final String ERROR_EXCEPTION_JSON = "exception"; + public static final String ERROR_CLASSNAME_JSON = "javaClassName"; + public static final String ERROR_MESSAGE_JSON = "message"; + + public static final String OP_PARAM = "op"; + public static final String DELEGATION_PARAM = "delegation"; + public static final String TOKEN_PARAM = "token"; + public static final String RENEWER_PARAM = "renewer"; + public static final String DELEGATION_TOKEN_URL = "url"; + public static final String DELEGATION_TOKEN_EXPIRATION_TIME = + "expirationTime"; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java new file mode 100644 index 00000000000..82e0d69c573 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenIdentifier.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; + +@Public +@Unstable +public class TimelineDelegationTokenIdentifier extends AbstractDelegationTokenIdentifier { + + public static final Text KIND_NAME = new Text("TIMELINE_DELEGATION_TOKEN"); + + public TimelineDelegationTokenIdentifier() { + + } + + /** + * Create a new timeline delegation token identifier + * + * @param owner the effective username of the token owner + * @param renewer the username of the renewer + * @param realUser the real username of the token owner + */ + public TimelineDelegationTokenIdentifier(Text owner, Text renewer, + Text realUser) { + super(owner, renewer, realUser); + } + + @Override + public Text getKind() { + return KIND_NAME; + } + + @InterfaceAudience.Private + public static class Renewer extends Token.TrivialRenewer { + @Override + protected Text getKind() { + return KIND_NAME; + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenOperation.java new file mode 100644 index 00000000000..33ab17829f0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenOperation.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPut; + +/** + * DelegationToken operations. + */ +@Unstable +@Private +public enum TimelineDelegationTokenOperation { + // TODO: need think about which ops can be done without kerberos + // credentials, for safety, we enforces all need kerberos credentials now. + GETDELEGATIONTOKEN(HttpGet.METHOD_NAME, true), + RENEWDELEGATIONTOKEN(HttpPut.METHOD_NAME, true), + CANCELDELEGATIONTOKEN(HttpPut.METHOD_NAME, true); + + private String httpMethod; + private boolean requiresKerberosCredentials; + + private TimelineDelegationTokenOperation(String httpMethod, + boolean requiresKerberosCredentials) { + this.httpMethod = httpMethod; + this.requiresKerberosCredentials = requiresKerberosCredentials; + } + + public String getHttpMethod() { + return httpMethod; + } + + public boolean requiresKerberosCredentials() { + return requiresKerberosCredentials; + } + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java new file mode 100644 index 00000000000..df1e84d5006 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/TimelineDelegationTokenSelector.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.security.client; + +import java.util.Collection; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenSelector; + +@Public +@Unstable +public class TimelineDelegationTokenSelector + implements TokenSelector { + + private static final Log LOG = LogFactory + .getLog(TimelineDelegationTokenSelector.class); + + @SuppressWarnings("unchecked") + public Token selectToken(Text service, + Collection> tokens) { + if (service == null) { + return null; + } + if (LOG.isDebugEnabled()) { + LOG.debug("Looking for a token with service " + service.toString()); + } + for (Token token : tokens) { + if (LOG.isDebugEnabled()) { + LOG.debug("Token kind is " + token.getKind().toString() + + " and the token's service name is " + token.getService()); + } + if (TimelineDelegationTokenIdentifier.KIND_NAME.equals(token.getKind()) + && service.equals(token.getService())) { + return (Token) token; + } + } + return null; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java index 324c6f663d5..a62ed4869da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java @@ -22,12 +22,10 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.codehaus.jackson.JsonGenerationException; -import org.codehaus.jackson.map.AnnotationIntrospector; import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; -import org.codehaus.jackson.xc.JaxbAnnotationIntrospector; /** * The helper class for the timeline module. @@ -41,9 +39,7 @@ public class TimelineUtils { static { mapper = new ObjectMapper(); - AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); - mapper.setAnnotationIntrospector(introspector); - mapper.setSerializationInclusion(Inclusion.NON_NULL); + YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java index ac0ea7f9832..3cc1aec5c6f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java @@ -49,9 +49,14 @@ public class YarnJacksonJaxbJsonProvider extends JacksonJaxbJsonProvider { @Override public ObjectMapper locateMapper(Class type, MediaType mediaType) { ObjectMapper mapper = super.locateMapper(type, mediaType); + configObjectMapper(mapper); + return mapper; + } + + public static void configObjectMapper(ObjectMapper mapper) { AnnotationIntrospector introspector = new JaxbAnnotationIntrospector(); mapper.setAnnotationIntrospector(introspector); mapper.setSerializationInclusion(Inclusion.NON_NULL); - return mapper; } + } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo index babc2fbf8e4..d90a26703ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo @@ -12,6 +12,7 @@ # limitations under the License. # org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo +org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo org.apache.hadoop.yarn.security.SchedulerSecurityInfo org.apache.hadoop.yarn.security.admin.AdminSecurityInfo diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier index d860461467e..a4ad54813d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -15,4 +15,5 @@ org.apache.hadoop.yarn.security.ContainerTokenIdentifier org.apache.hadoop.yarn.security.AMRMTokenIdentifier org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier +org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier org.apache.hadoop.yarn.security.NMTokenIdentifier diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer index dd0b2c48def..9fcfa19465f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer @@ -14,3 +14,4 @@ org.apache.hadoop.yarn.security.AMRMTokenIdentifier$Renewer org.apache.hadoop.yarn.security.ContainerTokenIdentifier$Renewer org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier$Renewer +org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier$Renewer diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 9f98834f7ce..a2f5a2489e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.Service; import org.apache.hadoop.util.ExitUtil; @@ -39,6 +40,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineAuthenticationFilterInitializer; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; @@ -56,10 +59,11 @@ public class ApplicationHistoryServer extends CompositeService { private static final Log LOG = LogFactory .getLog(ApplicationHistoryServer.class); - ApplicationHistoryClientService ahsClientService; - ApplicationHistoryManager historyManager; - TimelineStore timelineStore; - private WebApp webApp; + protected ApplicationHistoryClientService ahsClientService; + protected ApplicationHistoryManager historyManager; + protected TimelineStore timelineStore; + protected TimelineDelegationTokenSecretManagerService secretManagerService; + protected WebApp webApp; public ApplicationHistoryServer() { super(ApplicationHistoryServer.class.getName()); @@ -73,6 +77,8 @@ public class ApplicationHistoryServer extends CompositeService { addService((Service) historyManager); timelineStore = createTimelineStore(conf); addIfService(timelineStore); + secretManagerService = createTimelineDelegationTokenSecretManagerService(conf); + addService(secretManagerService); DefaultMetricsSystem.initialize("ApplicationHistoryServer"); JvmMetrics.initSingleton("ApplicationHistoryServer", null); @@ -158,21 +164,43 @@ public class ApplicationHistoryServer extends CompositeService { TimelineStore.class), conf); } + protected TimelineDelegationTokenSecretManagerService + createTimelineDelegationTokenSecretManagerService(Configuration conf) { + return new TimelineDelegationTokenSecretManagerService(); + } + protected void startWebApp() { - String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig()); + Configuration conf = getConfig(); + // Play trick to make the customized filter will only be loaded by the + // timeline server when security is enabled and Kerberos authentication + // is used. + if (UserGroupInformation.isSecurityEnabled() + && conf + .get(TimelineAuthenticationFilterInitializer.PREFIX + "type", "") + .equals("kerberos")) { + String initializers = conf.get("hadoop.http.filter.initializers"); + initializers = + initializers == null || initializers.length() == 0 ? "" : "," + + initializers; + if (!initializers.contains( + TimelineAuthenticationFilterInitializer.class.getName())) { + conf.set("hadoop.http.filter.initializers", + TimelineAuthenticationFilterInitializer.class.getName() + + initializers); + } + } + String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(conf); LOG.info("Instantiating AHSWebApp at " + bindAddress); try { + AHSWebApp ahsWebApp = AHSWebApp.getInstance(); + ahsWebApp.setApplicationHistoryManager(historyManager); + ahsWebApp.setTimelineStore(timelineStore); + ahsWebApp.setTimelineDelegationTokenSecretManagerService(secretManagerService); webApp = WebApps .$for("applicationhistory", ApplicationHistoryClientService.class, - ahsClientService, "ws") - .with(getConfig()) - .withHttpSpnegoPrincipalKey( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY) - .withHttpSpnegoKeytabKey( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) - .at(bindAddress) - .start(new AHSWebApp(historyManager, timelineStore)); + ahsClientService, "ws") + .with(conf).at(bindAddress).start(ahsWebApp); } catch (Exception e) { String msg = "AHSWebApp failed to start."; LOG.error(msg, e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java new file mode 100644 index 00000000000..53ef1ed3e84 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilter.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import java.util.Properties; + +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; + +@Private +@Unstable +public class TimelineAuthenticationFilter extends AuthenticationFilter { + + @Override + protected Properties getConfiguration(String configPrefix, + FilterConfig filterConfig) throws ServletException { + // In yarn-site.xml, we can simply set type to "kerberos". However, we need + // to replace the name here to use the customized Kerberos + DT service + // instead of the standard Kerberos handler. + Properties properties = super.getConfiguration(configPrefix, filterConfig); + if (properties.getProperty(AUTH_TYPE).equals("kerberos")) { + properties.setProperty( + AUTH_TYPE, TimelineClientAuthenticationService.class.getName()); + } + return properties; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java new file mode 100644 index 00000000000..e3c303233bb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineAuthenticationFilterInitializer.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import java.io.FileReader; +import java.io.IOException; +import java.io.Reader; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.FilterContainer; +import org.apache.hadoop.http.FilterInitializer; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.security.SecurityUtil; + +/** + *

+ * Initializes {@link TimelineAuthenticationFilter} which provides support for + * Kerberos HTTP SPNEGO authentication. + *

+ *

+ * It enables Kerberos HTTP SPNEGO plus delegation token authentication for the + * timeline server. + *

+ * Refer to the core-default.xml file, after the comment 'HTTP + * Authentication' for details on the configuration options. All related + * configuration properties have 'hadoop.http.authentication.' as prefix. + */ +public class TimelineAuthenticationFilterInitializer extends FilterInitializer { + + /** + * The configuration prefix of timeline Kerberos + DT authentication + */ + public static final String PREFIX = "yarn.timeline-service.http.authentication."; + + private static final String SIGNATURE_SECRET_FILE = + TimelineAuthenticationFilter.SIGNATURE_SECRET + ".file"; + + /** + *

+ * Initializes {@link TimelineAuthenticationFilter} + *

+ *

+ * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN + * configuration properties prefixed with + * "yarn.timeline-service.authentication." + *

+ * + * @param container + * The filter container + * @param conf + * Configuration for run-time parameters + */ + @Override + public void initFilter(FilterContainer container, Configuration conf) { + Map filterConfig = new HashMap(); + + // setting the cookie path to root '/' so it is used for all resources. + filterConfig.put(TimelineAuthenticationFilter.COOKIE_PATH, "/"); + + for (Map.Entry entry : conf) { + String name = entry.getKey(); + if (name.startsWith(PREFIX)) { + String value = conf.get(name); + name = name.substring(PREFIX.length()); + filterConfig.put(name, value); + } + } + + String signatureSecretFile = filterConfig.get(SIGNATURE_SECRET_FILE); + if (signatureSecretFile != null) { + try { + StringBuilder secret = new StringBuilder(); + Reader reader = new FileReader(signatureSecretFile); + int c = reader.read(); + while (c > -1) { + secret.append((char) c); + c = reader.read(); + } + reader.close(); + filterConfig + .put(TimelineAuthenticationFilter.SIGNATURE_SECRET, + secret.toString()); + } catch (IOException ex) { + throw new RuntimeException( + "Could not read HTTP signature secret file: " + + signatureSecretFile); + } + } + + // Resolve _HOST into bind address + String bindAddress = conf.get(HttpServer2.BIND_ADDRESS); + String principal = + filterConfig.get(TimelineClientAuthenticationService.PRINCIPAL); + if (principal != null) { + try { + principal = SecurityUtil.getServerPrincipal(principal, bindAddress); + } catch (IOException ex) { + throw new RuntimeException( + "Could not resolve Kerberos principal name: " + ex.toString(), ex); + } + filterConfig.put(TimelineClientAuthenticationService.PRINCIPAL, + principal); + } + + container.addGlobalFilter("Timeline Authentication Filter", + TimelineAuthenticationFilter.class.getName(), + filterConfig); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java new file mode 100644 index 00000000000..f11633d9697 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineClientAuthenticationService.java @@ -0,0 +1,236 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import java.io.IOException; +import java.io.Writer; +import java.text.MessageFormat; +import java.util.HashSet; +import java.util.Set; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.MediaType; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.server.AuthenticationToken; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenOperation; +import org.apache.hadoop.yarn.security.client.TimelineAuthenticationConsts; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; +import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; +import org.codehaus.jackson.map.ObjectMapper; + +/** + * Server side AuthenticationHandler that authenticates requests + * using the incoming delegation token as a 'delegation' query string parameter. + *

+ * If not delegation token is present in the request it delegates to the + * {@link KerberosAuthenticationHandler} + */ +@Private +@Unstable +public class TimelineClientAuthenticationService + extends KerberosAuthenticationHandler { + + public static final String TYPE = "kerberos-dt"; + private static final Set DELEGATION_TOKEN_OPS = new HashSet(); + private static final String OP_PARAM = "op"; + private static final String ENTER = System.getProperty("line.separator"); + + private ObjectMapper mapper; + + static { + DELEGATION_TOKEN_OPS.add( + TimelineDelegationTokenOperation.GETDELEGATIONTOKEN.toString()); + DELEGATION_TOKEN_OPS.add( + TimelineDelegationTokenOperation.RENEWDELEGATIONTOKEN.toString()); + DELEGATION_TOKEN_OPS.add( + TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN.toString()); + } + + public TimelineClientAuthenticationService() { + super(); + mapper = new ObjectMapper(); + YarnJacksonJaxbJsonProvider.configObjectMapper(mapper); + } + + /** + * Returns authentication type of the handler. + * + * @return delegationtoken-kerberos + */ + @Override + public String getType() { + return TYPE; + } + + @Override + public boolean managementOperation(AuthenticationToken token, + HttpServletRequest request, HttpServletResponse response) + throws IOException, AuthenticationException { + boolean requestContinues = true; + String op = request.getParameter(OP_PARAM); + op = (op != null) ? op.toUpperCase() : null; + if (DELEGATION_TOKEN_OPS.contains(op) && + !request.getMethod().equals("OPTIONS")) { + TimelineDelegationTokenOperation dtOp = + TimelineDelegationTokenOperation.valueOf(op); + if (dtOp.getHttpMethod().equals(request.getMethod())) { + if (dtOp.requiresKerberosCredentials() && token == null) { + response.sendError(HttpServletResponse.SC_UNAUTHORIZED, + MessageFormat.format( + "Operation [{0}] requires SPNEGO authentication established", + dtOp)); + requestContinues = false; + } else { + TimelineDelegationTokenSecretManagerService secretManager = + AHSWebApp.getInstance() + .getTimelineDelegationTokenSecretManagerService(); + try { + TimelineDelegationTokenResponse res = null; + switch (dtOp) { + case GETDELEGATIONTOKEN: + UserGroupInformation ownerUGI = + UserGroupInformation.createRemoteUser(token.getUserName()); + String renewerParam = + request + .getParameter(TimelineAuthenticationConsts.RENEWER_PARAM); + if (renewerParam == null) { + renewerParam = token.getUserName(); + } + Token dToken = + secretManager.createToken(ownerUGI, renewerParam); + res = new TimelineDelegationTokenResponse(); + res.setType(TimelineAuthenticationConsts.DELEGATION_TOKEN_URL); + res.setContent(dToken.encodeToUrlString()); + break; + case RENEWDELEGATIONTOKEN: + case CANCELDELEGATIONTOKEN: + String tokenParam = + request + .getParameter(TimelineAuthenticationConsts.TOKEN_PARAM); + if (tokenParam == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, + MessageFormat + .format( + "Operation [{0}] requires the parameter [{1}]", + dtOp, + TimelineAuthenticationConsts.TOKEN_PARAM)); + requestContinues = false; + } else { + if (dtOp == TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN) { + Token dt = + new Token(); + dt.decodeFromUrlString(tokenParam); + secretManager.cancelToken(dt, token.getUserName()); + } else { + Token dt = + new Token(); + dt.decodeFromUrlString(tokenParam); + long expirationTime = + secretManager.renewToken(dt, token.getUserName()); + res = new TimelineDelegationTokenResponse(); + res.setType(TimelineAuthenticationConsts.DELEGATION_TOKEN_EXPIRATION_TIME); + res.setContent(expirationTime); + } + } + break; + } + if (requestContinues) { + response.setStatus(HttpServletResponse.SC_OK); + if (res != null) { + response.setContentType(MediaType.APPLICATION_JSON); + Writer writer = response.getWriter(); + mapper.writeValue(writer, res); + writer.write(ENTER); + writer.flush(); + + } + requestContinues = false; + } + } catch (IOException e) { + throw new AuthenticationException(e.toString(), e); + } + } + } else { + response + .sendError( + HttpServletResponse.SC_BAD_REQUEST, + MessageFormat + .format( + "Wrong HTTP method [{0}] for operation [{1}], it should be [{2}]", + request.getMethod(), dtOp, dtOp.getHttpMethod())); + requestContinues = false; + } + } + return requestContinues; + } + + /** + * Authenticates a request looking for the delegation + * query-string parameter and verifying it is a valid token. If there is not + * delegation query-string parameter, it delegates the + * authentication to the {@link KerberosAuthenticationHandler} unless it is + * disabled. + * + * @param request + * the HTTP client request. + * @param response + * the HTTP client response. + * + * @return the authentication token for the authenticated request. + * @throws IOException + * thrown if an IO error occurred. + * @throws AuthenticationException + * thrown if the authentication failed. + */ + @Override + public AuthenticationToken authenticate(HttpServletRequest request, + HttpServletResponse response) + throws IOException, AuthenticationException { + AuthenticationToken token; + String delegationParam = + request + .getParameter(TimelineAuthenticationConsts.DELEGATION_PARAM); + if (delegationParam != null) { + Token dt = + new Token(); + dt.decodeFromUrlString(delegationParam); + TimelineDelegationTokenSecretManagerService secretManager = + AHSWebApp.getInstance() + .getTimelineDelegationTokenSecretManagerService(); + UserGroupInformation ugi = secretManager.verifyToken(dt); + final String shortName = ugi.getShortUserName(); + // creating a ephemeral token + token = new AuthenticationToken(shortName, ugi.getUserName(), getType()); + token.setExpires(0); + } else { + token = super.authenticate(request, response); + } + return token; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java new file mode 100644 index 00000000000..fee9eb41cd8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; + +/** + * The service wrapper of {@link TimelineDelegationTokenSecretManager} + */ +@Private +@Unstable +public class TimelineDelegationTokenSecretManagerService extends AbstractService { + + private TimelineDelegationTokenSecretManager secretManager = null; + private InetSocketAddress serviceAddr = null; + + public TimelineDelegationTokenSecretManagerService() { + super(TimelineDelegationTokenSecretManagerService.class.getName()); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + long secretKeyInterval = + conf.getLong(YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_KEY, + YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT); + long tokenMaxLifetime = + conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, + YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); + long tokenRenewInterval = + conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, + YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); + secretManager = new TimelineDelegationTokenSecretManager(secretKeyInterval, + tokenMaxLifetime, tokenRenewInterval, + 3600000); + secretManager.startThreads(); + + if (YarnConfiguration.useHttps(getConfig())) { + serviceAddr = getConfig().getSocketAddr( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT); + } else { + serviceAddr = getConfig().getSocketAddr( + YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT); + } + super.init(conf); + } + + @Override + protected void serviceStop() throws Exception { + secretManager.stopThreads(); + super.stop(); + } + + /** + * Creates a delegation token. + * + * @param ugi UGI creating the token. + * @param renewer token renewer. + * @return new delegation token. + * @throws IOException thrown if the token could not be created. + */ + public Token createToken( + UserGroupInformation ugi, String renewer) throws IOException { + renewer = (renewer == null) ? ugi.getShortUserName() : renewer; + String user = ugi.getUserName(); + Text owner = new Text(user); + Text realUser = null; + if (ugi.getRealUser() != null) { + realUser = new Text(ugi.getRealUser().getUserName()); + } + TimelineDelegationTokenIdentifier tokenIdentifier = + new TimelineDelegationTokenIdentifier(owner, new Text(renewer), realUser); + Token token = + new Token(tokenIdentifier, secretManager); + SecurityUtil.setTokenService(token, serviceAddr); + return token; + } + + /** + * Renews a delegation token. + * + * @param token delegation token to renew. + * @param renewer token renewer. + * @throws IOException thrown if the token could not be renewed. + */ + public long renewToken(Token token, + String renewer) throws IOException { + return secretManager.renewToken(token, renewer); + } + + /** + * Cancels a delegation token. + * + * @param token delegation token to cancel. + * @param canceler token canceler. + * @throws IOException thrown if the token could not be canceled. + */ + public void cancelToken(Token token, + String canceler) throws IOException { + secretManager.cancelToken(token, canceler); + } + + /** + * Verifies a delegation token. + * + * @param token delegation token to verify. + * @return the UGI for the token. + * @throws IOException thrown if the token could not be verified. + */ + public UserGroupInformation verifyToken(Token token) + throws IOException { + ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); + DataInputStream dis = new DataInputStream(buf); + TimelineDelegationTokenIdentifier id = new TimelineDelegationTokenIdentifier(); + try { + id.readFields(dis); + secretManager.verifyToken(id, token.getPassword()); + } finally { + dis.close(); + } + return id.getUser(); + } + + /** + * Create a timeline secret manager + * + * @param delegationKeyUpdateInterval + * the number of seconds for rolling new secret keys. + * @param delegationTokenMaxLifetime + * the maximum lifetime of the delegation tokens + * @param delegationTokenRenewInterval + * how often the tokens must be renewed + * @param delegationTokenRemoverScanInterval + * how often the tokens are scanned for expired tokens + */ + @Private + @Unstable + public static class TimelineDelegationTokenSecretManager extends + AbstractDelegationTokenSecretManager { + + public TimelineDelegationTokenSecretManager(long delegationKeyUpdateInterval, + long delegationTokenMaxLifetime, long delegationTokenRenewInterval, + long delegationTokenRemoverScanInterval) { + super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, + delegationTokenRenewInterval, delegationTokenRemoverScanInterval); + } + + @Override + public TimelineDelegationTokenIdentifier createIdentifier() { + return new TimelineDelegationTokenIdentifier(); + } + + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java index 93065b33681..c3a19d476b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java @@ -19,25 +19,70 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; import static org.apache.hadoop.yarn.util.StringHelper.pajoin; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.YarnWebParams; +import com.google.common.annotations.VisibleForTesting; + public class AHSWebApp extends WebApp implements YarnWebParams { - private final ApplicationHistoryManager applicationHistoryManager; - private final TimelineStore timelineStore; + private ApplicationHistoryManager applicationHistoryManager; + private TimelineStore timelineStore; + private TimelineDelegationTokenSecretManagerService secretManagerService; - public AHSWebApp(ApplicationHistoryManager applicationHistoryManager, - TimelineStore timelineStore) { + private static AHSWebApp instance = null; + + public static AHSWebApp getInstance() { + if (instance == null) { + instance = new AHSWebApp(); + } + return instance; + } + + @Private + @VisibleForTesting + public static void resetInstance() { + instance = null; + } + + private AHSWebApp() { + + } + + public ApplicationHistoryManager getApplicationHistoryManager() { + return applicationHistoryManager; + } + + public void setApplicationHistoryManager( + ApplicationHistoryManager applicationHistoryManager) { this.applicationHistoryManager = applicationHistoryManager; + } + + public TimelineStore getTimelineStore() { + return timelineStore; + } + + public void setTimelineStore(TimelineStore timelineStore) { this.timelineStore = timelineStore; } + public TimelineDelegationTokenSecretManagerService + getTimelineDelegationTokenSecretManagerService() { + return secretManagerService; + } + + public void setTimelineDelegationTokenSecretManagerService( + TimelineDelegationTokenSecretManagerService secretManagerService) { + this.secretManagerService = secretManagerService; + } + @Override public void setup() { bind(YarnJacksonJaxbJsonProvider.class); @@ -46,6 +91,8 @@ public class AHSWebApp extends WebApp implements YarnWebParams { bind(GenericExceptionHandler.class); bind(ApplicationContext.class).toInstance(applicationHistoryManager); bind(TimelineStore.class).toInstance(timelineStore); + bind(TimelineDelegationTokenSecretManagerService.class).toInstance( + secretManagerService); route("/", AHSController.class); route(pajoin("/apps", APP_STATE), AHSController.class); route(pajoin("/app", APPLICATION_ID), AHSController.class, "app"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index a35fe4600f9..3f3c08a55df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.junit.After; import org.junit.Before; @@ -74,6 +75,7 @@ public class TestApplicationHistoryClientService extends @After public void tearDown() throws Exception { + AHSWebApp.resetInstance(); historyServer.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java index d6d20af189c..5c55becb6c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.junit.After; import org.junit.Test; @@ -40,7 +41,7 @@ public class TestApplicationHistoryServer { Configuration config = new YarnConfiguration(); historyServer.init(config); assertEquals(STATE.INITED, historyServer.getServiceState()); - assertEquals(3, historyServer.getServices().size()); + assertEquals(4, historyServer.getServices().size()); ApplicationHistoryClientService historyService = historyServer.getClientService(); assertNotNull(historyServer.getClientService()); @@ -73,5 +74,6 @@ public class TestApplicationHistoryServer { if (historyServer != null) { historyServer.stop(); } + AHSWebApp.resetInstance(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 2baddf7b198..93d0a125f94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistor import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; @@ -719,6 +720,7 @@ public class MiniYARNCluster extends CompositeService { if (appHistoryServer != null) { appHistoryServer.stop(); } + AHSWebApp.resetInstance(); super.serviceStop(); } }