Revert "HADOOP-12563. Updated utility (dtutil) to create/modify token files. Contributed by Matthew Paduano"
This reverts commit 4838b735f0
.
This commit is contained in:
parent
5d4255a801
commit
d6402faded
|
@ -33,7 +33,6 @@ function hadoop_usage
|
|||
hadoop_add_subcommand "daemonlog" "get/set the log level for each daemon"
|
||||
hadoop_add_subcommand "distch" "distributed metadata changer"
|
||||
hadoop_add_subcommand "distcp" "copy file or directories recursively"
|
||||
hadoop_add_subcommand "dtutil" "operations related to delegation tokens"
|
||||
hadoop_add_subcommand "envvars" "display computed Hadoop environment variables"
|
||||
hadoop_add_subcommand "fs" "run a generic filesystem user client"
|
||||
hadoop_add_subcommand "jar <jar>" "run a jar file. NOTE: please use \"yarn jar\" to launch YARN applications, not this command."
|
||||
|
@ -140,9 +139,6 @@ case ${COMMAND} in
|
|||
CLASS=org.apache.hadoop.tools.DistCp
|
||||
hadoop_add_to_classpath_tools hadoop-distcp
|
||||
;;
|
||||
dtutil)
|
||||
CLASS=org.apache.hadoop.security.token.DtUtilShell
|
||||
;;
|
||||
envvars)
|
||||
echo "JAVA_HOME='${JAVA_HOME}'"
|
||||
echo "HADOOP_COMMON_HOME='${HADOOP_COMMON_HOME}'"
|
||||
|
|
|
@ -18,8 +18,6 @@
|
|||
|
||||
package org.apache.hadoop.security;
|
||||
|
||||
import com.google.protobuf.ByteString;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.DataInput;
|
||||
import java.io.DataInputStream;
|
||||
|
@ -27,7 +25,6 @@ import java.io.DataOutput;
|
|||
import java.io.DataOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
|
@ -50,8 +47,6 @@ import org.apache.hadoop.io.Writable;
|
|||
import org.apache.hadoop.io.WritableUtils;
|
||||
import org.apache.hadoop.security.token.Token;
|
||||
import org.apache.hadoop.security.token.TokenIdentifier;
|
||||
import org.apache.hadoop.security.proto.SecurityProtos.CredentialsKVProto;
|
||||
import org.apache.hadoop.security.proto.SecurityProtos.CredentialsProto;
|
||||
|
||||
/**
|
||||
* A class that provides the facilities of reading and writing
|
||||
|
@ -64,16 +59,16 @@ public class Credentials implements Writable {
|
|||
|
||||
private Map<Text, byte[]> secretKeysMap = new HashMap<Text, byte[]>();
|
||||
private Map<Text, Token<? extends TokenIdentifier>> tokenMap =
|
||||
new HashMap<Text, Token<? extends TokenIdentifier>>();
|
||||
new HashMap<Text, Token<? extends TokenIdentifier>>();
|
||||
|
||||
/**
|
||||
* Create an empty credentials instance.
|
||||
* Create an empty credentials instance
|
||||
*/
|
||||
public Credentials() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a copy of the given credentials.
|
||||
* Create a copy of the given credentials
|
||||
* @param credentials to copy
|
||||
*/
|
||||
public Credentials(Credentials credentials) {
|
||||
|
@ -81,7 +76,7 @@ public class Credentials implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Returns the Token object for the alias.
|
||||
* Returns the Token object for the alias
|
||||
* @param alias the alias for the Token
|
||||
* @return token for this alias
|
||||
*/
|
||||
|
@ -90,7 +85,7 @@ public class Credentials implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Add a token in the storage (in memory).
|
||||
* Add a token in the storage (in memory)
|
||||
* @param alias the alias for the key
|
||||
* @param t the token object
|
||||
*/
|
||||
|
@ -103,7 +98,7 @@ public class Credentials implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Return all the tokens in the in-memory map.
|
||||
* Return all the tokens in the in-memory map
|
||||
*/
|
||||
public Collection<Token<? extends TokenIdentifier>> getAllTokens() {
|
||||
return tokenMap.values();
|
||||
|
@ -117,7 +112,7 @@ public class Credentials implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Returns the key bytes for the alias.
|
||||
* Returns the key bytes for the alias
|
||||
* @param alias the alias for the key
|
||||
* @return key for this alias
|
||||
*/
|
||||
|
@ -133,7 +128,7 @@ public class Credentials implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Set the key for an alias.
|
||||
* Set the key for an alias
|
||||
* @param alias the alias for the key
|
||||
* @param key the key bytes
|
||||
*/
|
||||
|
@ -150,7 +145,7 @@ public class Credentials implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Return all the secret key entries in the in-memory map.
|
||||
* Return all the secret key entries in the in-memory map
|
||||
*/
|
||||
public List<Text> getAllSecretKeys() {
|
||||
List<Text> list = new java.util.ArrayList<Text>();
|
||||
|
@ -160,13 +155,13 @@ public class Credentials implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Convenience method for reading a token storage file and loading its Tokens.
|
||||
* Convenience method for reading a token storage file, and loading the Tokens
|
||||
* therein in the passed UGI
|
||||
* @param filename
|
||||
* @param conf
|
||||
* @throws IOException
|
||||
*/
|
||||
public static Credentials readTokenStorageFile(Path filename,
|
||||
Configuration conf)
|
||||
public static Credentials readTokenStorageFile(Path filename, Configuration conf)
|
||||
throws IOException {
|
||||
FSDataInputStream in = null;
|
||||
Credentials credentials = new Credentials();
|
||||
|
@ -183,13 +178,13 @@ public class Credentials implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Convenience method for reading a token storage file and loading its Tokens.
|
||||
* Convenience method for reading a token storage file, and loading the Tokens
|
||||
* therein in the passed UGI
|
||||
* @param filename
|
||||
* @param conf
|
||||
* @throws IOException
|
||||
*/
|
||||
public static Credentials readTokenStorageFile(File filename,
|
||||
Configuration conf)
|
||||
public static Credentials readTokenStorageFile(File filename, Configuration conf)
|
||||
throws IOException {
|
||||
DataInputStream in = null;
|
||||
Credentials credentials = new Credentials();
|
||||
|
@ -206,7 +201,8 @@ public class Credentials implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Convenience method for reading a token from a DataInputStream.
|
||||
* Convenience method for reading a token storage file directly from a
|
||||
* datainputstream
|
||||
*/
|
||||
public void readTokenStorageStream(DataInputStream in) throws IOException {
|
||||
byte[] magic = new byte[TOKEN_STORAGE_MAGIC.length];
|
||||
|
@ -215,33 +211,22 @@ public class Credentials implements Writable {
|
|||
throw new IOException("Bad header found in token storage.");
|
||||
}
|
||||
byte version = in.readByte();
|
||||
if (version != TOKEN_STORAGE_VERSION &&
|
||||
version != OLD_TOKEN_STORAGE_VERSION) {
|
||||
if (version != TOKEN_STORAGE_VERSION) {
|
||||
throw new IOException("Unknown version " + version +
|
||||
" in token storage.");
|
||||
}
|
||||
if (version == OLD_TOKEN_STORAGE_VERSION) {
|
||||
readFields(in);
|
||||
} else if (version == TOKEN_STORAGE_VERSION) {
|
||||
readProtos(in);
|
||||
}
|
||||
readFields(in);
|
||||
}
|
||||
|
||||
private static final byte[] TOKEN_STORAGE_MAGIC =
|
||||
"HDTS".getBytes(Charsets.UTF_8);
|
||||
private static final byte TOKEN_STORAGE_VERSION = 1;
|
||||
|
||||
/**
|
||||
* For backward compatibility.
|
||||
*/
|
||||
private static final byte OLD_TOKEN_STORAGE_VERSION = 0;
|
||||
|
||||
private static final byte TOKEN_STORAGE_VERSION = 0;
|
||||
|
||||
public void writeTokenStorageToStream(DataOutputStream os)
|
||||
throws IOException {
|
||||
throws IOException {
|
||||
os.write(TOKEN_STORAGE_MAGIC);
|
||||
os.write(TOKEN_STORAGE_VERSION);
|
||||
writeProto(os);
|
||||
write(os);
|
||||
}
|
||||
|
||||
public void writeTokenStorageFile(Path filename,
|
||||
|
@ -252,29 +237,7 @@ public class Credentials implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* For backward compatibility.
|
||||
*/
|
||||
public void writeLegacyTokenStorageLocalFile(File f) throws IOException {
|
||||
writeLegacyOutputStream(new DataOutputStream(new FileOutputStream(f)));
|
||||
}
|
||||
|
||||
/**
|
||||
* For backward compatibility.
|
||||
*/
|
||||
public void writeLegacyTokenStorageFile(Path filename, Configuration conf)
|
||||
throws IOException {
|
||||
writeLegacyOutputStream(filename.getFileSystem(conf).create(filename));
|
||||
}
|
||||
|
||||
private void writeLegacyOutputStream(DataOutputStream os) throws IOException {
|
||||
os.write(TOKEN_STORAGE_MAGIC);
|
||||
os.write(OLD_TOKEN_STORAGE_VERSION);
|
||||
write(os);
|
||||
os.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores all the keys to DataOutput.
|
||||
* Stores all the keys to DataOutput
|
||||
* @param out
|
||||
* @throws IOException
|
||||
*/
|
||||
|
@ -283,7 +246,7 @@ public class Credentials implements Writable {
|
|||
// write out tokens first
|
||||
WritableUtils.writeVInt(out, tokenMap.size());
|
||||
for(Map.Entry<Text,
|
||||
Token<? extends TokenIdentifier>> e: tokenMap.entrySet()) {
|
||||
Token<? extends TokenIdentifier>> e: tokenMap.entrySet()) {
|
||||
e.getKey().write(out);
|
||||
e.getValue().write(out);
|
||||
}
|
||||
|
@ -298,49 +261,7 @@ public class Credentials implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Write contents of this instance as CredentialsProto message to DataOutput.
|
||||
* @param out
|
||||
* @throws IOException
|
||||
*/
|
||||
public void writeProto(DataOutput out) throws IOException {
|
||||
CredentialsProto.Builder storage = CredentialsProto.newBuilder();
|
||||
for (Map.Entry<Text, Token<? extends TokenIdentifier>> e :
|
||||
tokenMap.entrySet()) {
|
||||
CredentialsKVProto.Builder kv = CredentialsKVProto.newBuilder().
|
||||
setAliasBytes(ByteString.copyFrom(
|
||||
e.getKey().getBytes(), 0, e.getKey().getLength())).
|
||||
setToken(e.getValue().toTokenProto());
|
||||
storage.addTokens(kv.build());
|
||||
}
|
||||
|
||||
for(Map.Entry<Text, byte[]> e : secretKeysMap.entrySet()) {
|
||||
CredentialsKVProto.Builder kv = CredentialsKVProto.newBuilder().
|
||||
setAliasBytes(ByteString.copyFrom(
|
||||
e.getKey().getBytes(), 0, e.getKey().getLength())).
|
||||
setSecret(ByteString.copyFrom(e.getValue()));
|
||||
storage.addSecrets(kv.build());
|
||||
}
|
||||
storage.build().writeTo((DataOutputStream)out);
|
||||
}
|
||||
|
||||
/**
|
||||
* Populates keys/values from proto buffer storage.
|
||||
* @param in - stream ready to read a serialized proto buffer message
|
||||
*/
|
||||
public void readProtos(DataInput in) throws IOException {
|
||||
CredentialsProto storage = CredentialsProto.parseFrom((DataInputStream)in);
|
||||
for (CredentialsKVProto kv : storage.getTokensList()) {
|
||||
addToken(new Text(kv.getAliasBytes().toByteArray()),
|
||||
(Token<? extends TokenIdentifier>) new Token(kv.getToken()));
|
||||
}
|
||||
for (CredentialsKVProto kv : storage.getSecretsList()) {
|
||||
addSecretKey(new Text(kv.getAliasBytes().toByteArray()),
|
||||
kv.getSecret().toByteArray());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads all the keys.
|
||||
* Loads all the keys
|
||||
* @param in
|
||||
* @throws IOException
|
||||
*/
|
||||
|
|
|
@ -1,41 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.security.token;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.io.Text;
|
||||
import org.apache.hadoop.security.Credentials;
|
||||
|
||||
/**
|
||||
* DtFetcher is an interface which permits the abstraction and separation of
|
||||
* delegation token fetch implementaions across different packages and
|
||||
* compilation units. Resolution of fetcher impl will be done at runtime.
|
||||
*/
|
||||
public interface DtFetcher {
|
||||
/** Return a key used to identify the object/service implementation. */
|
||||
Text getServiceName();
|
||||
|
||||
/** Used to allow the service API to indicate whether a token is required. */
|
||||
boolean isTokenRequired();
|
||||
|
||||
/** Add any number of delegation tokens to Credentials object and return
|
||||
* a token instance that is appropriate for aliasing, or null if none. */
|
||||
Token<?> addDelegationTokens(Configuration conf, Credentials creds,
|
||||
String renewer, String url) throws Exception;
|
||||
}
|
|
@ -1,271 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.security.token;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
import java.text.DateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.ServiceLoader;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.io.Text;
|
||||
import org.apache.hadoop.security.Credentials;
|
||||
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
|
||||
|
||||
/**
|
||||
* DtFileOperations is a collection of delegation token file operations.
|
||||
*/
|
||||
public final class DtFileOperations {
|
||||
private static final Log LOG = LogFactory.getLog(DtFileOperations.class);
|
||||
|
||||
/** No public constructor as per checkstyle. */
|
||||
private DtFileOperations() { }
|
||||
|
||||
/**
|
||||
* Use FORMAT_* as arguments to format parameters.
|
||||
* FORMAT_PB is for protobuf output.
|
||||
*/
|
||||
public static final String FORMAT_PB = "protobuf";
|
||||
|
||||
/**
|
||||
* Use FORMAT_* as arguments to format parameters.
|
||||
* FORMAT_JAVA is a legacy option for java serialization output.
|
||||
*/
|
||||
public static final String FORMAT_JAVA = "java";
|
||||
|
||||
private static final String NA_STRING = "-NA-";
|
||||
private static final String PREFIX_HTTP = "http://";
|
||||
private static final String PREFIX_HTTPS = "https://";
|
||||
|
||||
/** Let the DtFetcher code add the appropriate prefix if HTTP/S is used. */
|
||||
private static String stripPrefix(String u) {
|
||||
return u.replaceFirst(PREFIX_HTTP, "").replaceFirst(PREFIX_HTTPS, "");
|
||||
}
|
||||
|
||||
/** Match token service field to alias text. True if alias is null. */
|
||||
private static boolean matchAlias(Token<?> token, Text alias) {
|
||||
return alias == null || token.getService().equals(alias);
|
||||
}
|
||||
|
||||
/** Match fetcher's service name to the service text and/or url prefix. */
|
||||
private static boolean matchService(
|
||||
DtFetcher fetcher, Text service, String url) {
|
||||
Text sName = fetcher.getServiceName();
|
||||
return (service == null && url.startsWith(sName.toString() + "://")) ||
|
||||
(service != null && service.equals(sName));
|
||||
}
|
||||
|
||||
/** Format a long integer type into a date string. */
|
||||
private static String formatDate(long date) {
|
||||
DateFormat df = DateFormat.getDateTimeInstance(
|
||||
DateFormat.SHORT, DateFormat.SHORT);
|
||||
return df.format(new Date(date));
|
||||
}
|
||||
|
||||
/** Add the service prefix for a local filesystem. */
|
||||
private static Path fileToPath(File f) {
|
||||
return new Path("file:" + f.getAbsolutePath());
|
||||
}
|
||||
|
||||
/** Write out a Credentials object as a local file.
|
||||
* @param f a local File object.
|
||||
* @param format a string equal to FORMAT_PB or FORMAT_JAVA.
|
||||
* @param creds the Credentials object to be written out.
|
||||
* @param conf a Configuration object passed along.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void doFormattedWrite(
|
||||
File f, String format, Credentials creds, Configuration conf)
|
||||
throws IOException {
|
||||
if (format == null || format.equals(FORMAT_PB)) {
|
||||
creds.writeTokenStorageFile(fileToPath(f), conf);
|
||||
} else { // if (format != null && format.equals(FORMAT_JAVA)) {
|
||||
creds.writeLegacyTokenStorageLocalFile(f);
|
||||
}
|
||||
}
|
||||
|
||||
/** Print out a Credentials file from the local filesystem.
|
||||
* @param tokenFile a local File object.
|
||||
* @param alias print only tokens matching alias (null matches all).
|
||||
* @param conf Configuration object passed along.
|
||||
* @param out print to this stream.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void printTokenFile(
|
||||
File tokenFile, Text alias, Configuration conf, PrintStream out)
|
||||
throws IOException {
|
||||
out.println("File: " + tokenFile.getPath());
|
||||
Credentials creds = Credentials.readTokenStorageFile(tokenFile, conf);
|
||||
printCredentials(creds, alias, out);
|
||||
}
|
||||
|
||||
/** Print out a Credentials object.
|
||||
* @param creds the Credentials object to be printed out.
|
||||
* @param alias print only tokens matching alias (null matches all).
|
||||
* @param out print to this stream.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void printCredentials(
|
||||
Credentials creds, Text alias, PrintStream out)
|
||||
throws IOException {
|
||||
boolean tokenHeader = true;
|
||||
String fmt = "%-24s %-20s %-15s %-12s %s%n";
|
||||
for (Token<?> token : creds.getAllTokens()) {
|
||||
if (matchAlias(token, alias)) {
|
||||
if (tokenHeader) {
|
||||
out.printf(fmt, "Token kind", "Service", "Renewer", "Exp date",
|
||||
"URL enc token");
|
||||
out.println(StringUtils.repeat("-", 80));
|
||||
tokenHeader = false;
|
||||
}
|
||||
AbstractDelegationTokenIdentifier id =
|
||||
(AbstractDelegationTokenIdentifier) token.decodeIdentifier();
|
||||
out.printf(fmt, token.getKind(), token.getService(),
|
||||
(id != null) ? id.getRenewer() : NA_STRING,
|
||||
(id != null) ? formatDate(id.getMaxDate()) : NA_STRING,
|
||||
token.encodeToUrlString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Fetch a token from a service and save to file in the local filesystem.
|
||||
* @param tokenFile a local File object to hold the output.
|
||||
* @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output
|
||||
* @param alias overwrite service field of fetched token with this text.
|
||||
* @param service use a DtFetcher implementation matching this service text.
|
||||
* @param url pass this URL to fetcher after stripping any http/s prefix.
|
||||
* @param renewer pass this renewer to the fetcher.
|
||||
* @param conf Configuration object passed along.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void getTokenFile(File tokenFile, String fileFormat,
|
||||
Text alias, Text service, String url, String renewer, Configuration conf)
|
||||
throws Exception {
|
||||
Token<?> token = null;
|
||||
Credentials creds = tokenFile.exists() ?
|
||||
Credentials.readTokenStorageFile(tokenFile, conf) : new Credentials();
|
||||
ServiceLoader<DtFetcher> loader = ServiceLoader.load(DtFetcher.class);
|
||||
for (DtFetcher fetcher : loader) {
|
||||
if (matchService(fetcher, service, url)) {
|
||||
if (!fetcher.isTokenRequired()) {
|
||||
String message = "DtFetcher for service '" + service +
|
||||
"' does not require a token. Check your configuration. " +
|
||||
"Note: security may be disabled or there may be two DtFetcher " +
|
||||
"providers for the same service designation.";
|
||||
LOG.error(message);
|
||||
throw new IllegalArgumentException(message);
|
||||
}
|
||||
token = fetcher.addDelegationTokens(conf, creds, renewer,
|
||||
stripPrefix(url));
|
||||
}
|
||||
}
|
||||
if (alias != null) {
|
||||
if (token == null) {
|
||||
String message = "DtFetcher for service '" + service + "'" +
|
||||
" does not allow aliasing. Cannot apply alias '" + alias + "'." +
|
||||
" Drop alias flag to get token for this service.";
|
||||
LOG.error(message);
|
||||
throw new IOException(message);
|
||||
}
|
||||
Token<?> aliasedToken = token.copyToken();
|
||||
aliasedToken.setService(alias);
|
||||
creds.addToken(alias, aliasedToken);
|
||||
LOG.info("Add token with service " + alias);
|
||||
}
|
||||
doFormattedWrite(tokenFile, fileFormat, creds, conf);
|
||||
}
|
||||
|
||||
/** Append tokens from list of files in local filesystem, saving to last file.
|
||||
* @param tokenFiles list of local File objects. Last file holds the output.
|
||||
* @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output
|
||||
* @param conf Configuration object passed along.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void appendTokenFiles(
|
||||
ArrayList<File> tokenFiles, String fileFormat, Configuration conf)
|
||||
throws IOException {
|
||||
Credentials newCreds = new Credentials();
|
||||
File lastTokenFile = null;
|
||||
for (File tokenFile : tokenFiles) {
|
||||
lastTokenFile = tokenFile;
|
||||
Credentials creds = Credentials.readTokenStorageFile(tokenFile, conf);
|
||||
for (Token<?> token : creds.getAllTokens()) {
|
||||
newCreds.addToken(token.getService(), token);
|
||||
}
|
||||
}
|
||||
doFormattedWrite(lastTokenFile, fileFormat, newCreds, conf);
|
||||
}
|
||||
|
||||
/** Remove a token from a file in the local filesystem, matching alias.
|
||||
* @param cancel cancel token as well as remove from file.
|
||||
* @param tokenFile a local File object.
|
||||
* @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output
|
||||
* @param alias remove only tokens matching alias; null matches all.
|
||||
* @param conf Configuration object passed along.
|
||||
* @throws IOException
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public static void removeTokenFromFile(boolean cancel,
|
||||
File tokenFile, String fileFormat, Text alias, Configuration conf)
|
||||
throws IOException, InterruptedException {
|
||||
Credentials newCreds = new Credentials();
|
||||
Credentials creds = Credentials.readTokenStorageFile(tokenFile, conf);
|
||||
for (Token<?> token : creds.getAllTokens()) {
|
||||
if (matchAlias(token, alias)) {
|
||||
if (token.isManaged() && cancel) {
|
||||
token.cancel(conf);
|
||||
LOG.info("Canceled " + token.getKind() + ":" + token.getService());
|
||||
}
|
||||
} else {
|
||||
newCreds.addToken(token.getService(), token);
|
||||
}
|
||||
}
|
||||
doFormattedWrite(tokenFile, fileFormat, newCreds, conf);
|
||||
}
|
||||
|
||||
/** Renew a token from a file in the local filesystem, matching alias.
|
||||
* @param tokenFile a local File object.
|
||||
* @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output
|
||||
* @param alias renew only tokens matching alias; null matches all.
|
||||
* @param conf Configuration object passed along.
|
||||
* @throws IOException
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public static void renewTokenFile(
|
||||
File tokenFile, String fileFormat, Text alias, Configuration conf)
|
||||
throws IOException, InterruptedException {
|
||||
Credentials creds = Credentials.readTokenStorageFile(tokenFile, conf);
|
||||
for (Token<?> token : creds.getAllTokens()) {
|
||||
if (token.isManaged() && matchAlias(token, alias)) {
|
||||
long result = token.renew(conf);
|
||||
LOG.info("Renewed" + token.getKind() + ":" + token.getService() +
|
||||
" until " + formatDate(result));
|
||||
}
|
||||
}
|
||||
doFormattedWrite(tokenFile, fileFormat, creds, conf);
|
||||
}
|
||||
}
|
|
@ -1,326 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.security.token;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.io.Text;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.tools.CommandShell;
|
||||
import org.apache.hadoop.util.ToolRunner;
|
||||
|
||||
/**
|
||||
* DtUtilShell is a set of command line token file management operations.
|
||||
*/
|
||||
public class DtUtilShell extends CommandShell {
|
||||
private static final Log LOG = LogFactory.getLog(DtUtilShell.class);
|
||||
|
||||
private static final String FORMAT_SUBSTRING = "[-format (" +
|
||||
DtFileOperations.FORMAT_JAVA + "|" +
|
||||
DtFileOperations.FORMAT_PB + ")]";
|
||||
public static final String DT_USAGE = "hadoop dtutil " +
|
||||
"[-keytab <keytab_file> -principal <principal_name>] " +
|
||||
"subcommand (help|print|get|append|cancel|remove|renew) " +
|
||||
FORMAT_SUBSTRING + " [-alias <alias>] filename...";
|
||||
|
||||
// command line options
|
||||
private static final String HELP = "help";
|
||||
private static final String KEYTAB = "-keytab";
|
||||
private static final String PRINCIPAL = "-principal";
|
||||
private static final String PRINT = "print";
|
||||
private static final String GET = "get";
|
||||
private static final String APPEND = "append";
|
||||
private static final String CANCEL = "cancel";
|
||||
private static final String REMOVE = "remove";
|
||||
private static final String RENEW = "renew";
|
||||
private static final String RENEWER = "-renewer";
|
||||
private static final String SERVICE = "-service";
|
||||
private static final String ALIAS = "-alias";
|
||||
private static final String FORMAT = "-format";
|
||||
|
||||
// configuration state from args, conf
|
||||
private String keytab = null;
|
||||
private String principal = null;
|
||||
private Text alias = null;
|
||||
private Text service = null;
|
||||
private String renewer = null;
|
||||
private String format = DtFileOperations.FORMAT_PB;
|
||||
private ArrayList<File> tokenFiles = null;
|
||||
private File firstFile = null;
|
||||
|
||||
/**
|
||||
* Parse arguments looking for Kerberos keytab/principal.
|
||||
* If both are found: remove both from the argument list and attempt login.
|
||||
* If only one of the two is found: remove it from argument list, log warning
|
||||
* and do not attempt login.
|
||||
* If neither is found: return original args array, doing nothing.
|
||||
* Return the pruned args array if either flag is present.
|
||||
*/
|
||||
private String[] maybeDoLoginFromKeytabAndPrincipal(String[] args)
|
||||
throws IOException{
|
||||
ArrayList<String> savedArgs = new ArrayList<String>(args.length);
|
||||
for (int i = 0; i < args.length; i++) {
|
||||
String current = args[i];
|
||||
if (current.equals(PRINCIPAL)) {
|
||||
principal = args[++i];
|
||||
} else if (current.equals(KEYTAB)) {
|
||||
keytab = args[++i];
|
||||
} else {
|
||||
savedArgs.add(current);
|
||||
}
|
||||
}
|
||||
int newSize = savedArgs.size();
|
||||
if (newSize != args.length) {
|
||||
if (principal != null && keytab != null) {
|
||||
UserGroupInformation.loginUserFromKeytab(principal, keytab);
|
||||
} else {
|
||||
LOG.warn("-principal and -keytab not both specified! " +
|
||||
"Kerberos login not attempted.");
|
||||
}
|
||||
return savedArgs.toArray(new String[newSize]);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the command line arguments and initialize subcommand.
|
||||
* Also will attempt to perform Kerberos login if both -principal and -keytab
|
||||
* flags are passed in args array.
|
||||
* @param args
|
||||
* @return 0 if the argument(s) were recognized, 1 otherwise
|
||||
* @throws Exception
|
||||
*/
|
||||
@Override
|
||||
protected int init(String[] args) throws Exception {
|
||||
if (0 == args.length) {
|
||||
return 1;
|
||||
}
|
||||
tokenFiles = new ArrayList<File>();
|
||||
args = maybeDoLoginFromKeytabAndPrincipal(args);
|
||||
for (int i = 0; i < args.length; i++) {
|
||||
if (i == 0) {
|
||||
String command = args[0];
|
||||
if (command.equals(HELP)) {
|
||||
return 1;
|
||||
} else if (command.equals(PRINT)) {
|
||||
setSubCommand(new Print());
|
||||
} else if (command.equals(GET)) {
|
||||
setSubCommand(new Get(args[++i]));
|
||||
} else if (command.equals(APPEND)) {
|
||||
setSubCommand(new Append());
|
||||
} else if (command.equals(CANCEL)) {
|
||||
setSubCommand(new Remove(true));
|
||||
} else if (command.equals(REMOVE)) {
|
||||
setSubCommand(new Remove(false));
|
||||
} else if (command.equals(RENEW)) {
|
||||
setSubCommand(new Renew());
|
||||
}
|
||||
} else if (args[i].equals(ALIAS)) {
|
||||
alias = new Text(args[++i]);
|
||||
} else if (args[i].equals(SERVICE)) {
|
||||
service = new Text(args[++i]);
|
||||
} else if (args[i].equals(RENEWER)) {
|
||||
renewer = args[++i];
|
||||
} else if (args[i].equals(FORMAT)) {
|
||||
format = args[++i];
|
||||
if (!format.equals(DtFileOperations.FORMAT_JAVA) &&
|
||||
!format.equals(DtFileOperations.FORMAT_PB)) {
|
||||
LOG.error("-format must be '" + DtFileOperations.FORMAT_JAVA +
|
||||
"' or '" + DtFileOperations.FORMAT_PB + "' not '" +
|
||||
format + "'");
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
for (; i < args.length; i++) {
|
||||
File f = new File(args[i]);
|
||||
if (f.exists()) {
|
||||
tokenFiles.add(f);
|
||||
}
|
||||
if (firstFile == null) {
|
||||
firstFile = f;
|
||||
}
|
||||
}
|
||||
if (tokenFiles.size() == 0 && firstFile == null) {
|
||||
LOG.error("Must provide a filename to all commands.");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getCommandUsage() {
|
||||
return String.format("%n%s%n %s%n %s%n %s%n %s%n %s%n %s%n%n",
|
||||
DT_USAGE, (new Print()).getUsage(), (new Get()).getUsage(),
|
||||
(new Append()).getUsage(), (new Remove(true)).getUsage(),
|
||||
(new Remove(false)).getUsage(), (new Renew()).getUsage());
|
||||
}
|
||||
|
||||
private class Print extends SubCommand {
|
||||
public static final String PRINT_USAGE =
|
||||
"dtutil print [-alias <alias>] filename...";
|
||||
|
||||
@Override
|
||||
public void execute() throws Exception {
|
||||
for (File tokenFile : tokenFiles) {
|
||||
DtFileOperations.printTokenFile(tokenFile, alias, getConf(), getOut());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsage() {
|
||||
return PRINT_USAGE;
|
||||
}
|
||||
}
|
||||
|
||||
private class Get extends SubCommand {
|
||||
public static final String GET_USAGE = "dtutil get URL " +
|
||||
"[-service <scheme>] " + FORMAT_SUBSTRING +
|
||||
"[-alias <alias>] [-renewer <renewer>] filename";
|
||||
private static final String PREFIX_HTTP = "http://";
|
||||
private static final String PREFIX_HTTPS = "https://";
|
||||
|
||||
private String url = null;
|
||||
|
||||
public Get() { }
|
||||
|
||||
public Get(String arg) {
|
||||
url = arg;
|
||||
}
|
||||
|
||||
public boolean isGenericUrl() {
|
||||
return url.startsWith(PREFIX_HTTP) || url.startsWith(PREFIX_HTTPS);
|
||||
}
|
||||
|
||||
public boolean validate() {
|
||||
if (service != null && !isGenericUrl()) {
|
||||
LOG.error("Only provide -service with http/https URL.");
|
||||
return false;
|
||||
}
|
||||
if (service == null && isGenericUrl()) {
|
||||
LOG.error("Must provide -service with http/https URL.");
|
||||
return false;
|
||||
}
|
||||
if (url.indexOf("://") == -1) {
|
||||
LOG.error("URL does not contain a service specification: " + url);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute() throws Exception {
|
||||
DtFileOperations.getTokenFile(
|
||||
firstFile, format, alias, service, url, renewer, getConf());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsage() {
|
||||
return GET_USAGE;
|
||||
}
|
||||
}
|
||||
|
||||
private class Append extends SubCommand {
|
||||
public static final String APPEND_USAGE =
|
||||
"dtutil append " + FORMAT_SUBSTRING + "filename...";
|
||||
|
||||
@Override
|
||||
public void execute() throws Exception {
|
||||
DtFileOperations.appendTokenFiles(tokenFiles, format, getConf());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsage() {
|
||||
return APPEND_USAGE;
|
||||
}
|
||||
}
|
||||
|
||||
private class Remove extends SubCommand {
|
||||
public static final String REMOVE_USAGE =
|
||||
"dtutil remove -alias <alias> " + FORMAT_SUBSTRING + " filename...";
|
||||
public static final String CANCEL_USAGE =
|
||||
"dtutil cancel -alias <alias> " + FORMAT_SUBSTRING + " filename...";
|
||||
private boolean cancel = false;
|
||||
|
||||
public Remove(boolean arg) {
|
||||
cancel = arg;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean validate() {
|
||||
if (alias == null) {
|
||||
LOG.error("-alias flag is not optional for remove or cancel");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute() throws Exception {
|
||||
for (File tokenFile : tokenFiles) {
|
||||
DtFileOperations.removeTokenFromFile(
|
||||
cancel, tokenFile, format, alias, getConf());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsage() {
|
||||
if (cancel) {
|
||||
return CANCEL_USAGE;
|
||||
}
|
||||
return REMOVE_USAGE;
|
||||
}
|
||||
}
|
||||
|
||||
private class Renew extends SubCommand {
|
||||
public static final String RENEW_USAGE =
|
||||
"dtutil renew -alias <alias> filename...";
|
||||
|
||||
@Override
|
||||
public boolean validate() {
|
||||
if (alias == null) {
|
||||
LOG.error("-alias flag is not optional for renew");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void execute() throws Exception {
|
||||
for (File tokenFile : tokenFiles) {
|
||||
DtFileOperations.renewTokenFile(tokenFile, format, alias, getConf());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUsage() {
|
||||
return RENEW_USAGE;
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
System.exit(ToolRunner.run(new Configuration(), new DtUtilShell(), args));
|
||||
}
|
||||
}
|
|
@ -19,7 +19,6 @@
|
|||
package org.apache.hadoop.security.token;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.protobuf.ByteString;
|
||||
import com.google.common.primitives.Bytes;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
|
@ -29,7 +28,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
|||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.io.*;
|
||||
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
|
||||
import org.apache.hadoop.util.ReflectionUtils;
|
||||
|
||||
import java.io.*;
|
||||
|
@ -82,7 +80,7 @@ public class Token<T extends TokenIdentifier> implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Default constructor.
|
||||
* Default constructor
|
||||
*/
|
||||
public Token() {
|
||||
identifier = new byte[0];
|
||||
|
@ -102,38 +100,8 @@ public class Token<T extends TokenIdentifier> implements Writable {
|
|||
this.service = other.service;
|
||||
}
|
||||
|
||||
public Token<T> copyToken() {
|
||||
return new Token<T>(this);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a Token from a TokenProto.
|
||||
* @param tokenPB the TokenProto object
|
||||
*/
|
||||
public Token(TokenProto tokenPB) {
|
||||
this.identifier = tokenPB.getIdentifier().toByteArray();
|
||||
this.password = tokenPB.getPassword().toByteArray();
|
||||
this.kind = new Text(tokenPB.getKindBytes().toByteArray());
|
||||
this.service = new Text(tokenPB.getServiceBytes().toByteArray());
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a TokenProto from this Token instance.
|
||||
* @return a new TokenProto object holding copies of data in this instance
|
||||
*/
|
||||
public TokenProto toTokenProto() {
|
||||
return TokenProto.newBuilder().
|
||||
setIdentifier(ByteString.copyFrom(this.getIdentifier())).
|
||||
setPassword(ByteString.copyFrom(this.getPassword())).
|
||||
setKindBytes(ByteString.copyFrom(
|
||||
this.getKind().getBytes(), 0, this.getKind().getLength())).
|
||||
setServiceBytes(ByteString.copyFrom(
|
||||
this.getService().getBytes(), 0, this.getService().getLength())).
|
||||
build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the token identifier's byte representation.
|
||||
* Get the token identifier's byte representation
|
||||
* @return the token identifier's byte representation
|
||||
*/
|
||||
public byte[] getIdentifier() {
|
||||
|
@ -180,7 +148,7 @@ public class Token<T extends TokenIdentifier> implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get the token password/secret.
|
||||
* Get the token password/secret
|
||||
* @return the token password/secret
|
||||
*/
|
||||
public byte[] getPassword() {
|
||||
|
@ -188,7 +156,7 @@ public class Token<T extends TokenIdentifier> implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get the token kind.
|
||||
* Get the token kind
|
||||
* @return the kind of the token
|
||||
*/
|
||||
public synchronized Text getKind() {
|
||||
|
@ -207,7 +175,7 @@ public class Token<T extends TokenIdentifier> implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get the service on which the token is supposed to be used.
|
||||
* Get the service on which the token is supposed to be used
|
||||
* @return the service name
|
||||
*/
|
||||
public Text getService() {
|
||||
|
@ -215,7 +183,7 @@ public class Token<T extends TokenIdentifier> implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Set the service on which the token is supposed to be used.
|
||||
* Set the service on which the token is supposed to be used
|
||||
* @param newService the service name
|
||||
*/
|
||||
public void setService(Text newService) {
|
||||
|
@ -278,7 +246,7 @@ public class Token<T extends TokenIdentifier> implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Modify the writable to the value from the newValue.
|
||||
* Modify the writable to the value from the newValue
|
||||
* @param obj the object to read into
|
||||
* @param newValue the string with the url-safe base64 encoded bytes
|
||||
* @throws IOException
|
||||
|
@ -293,7 +261,7 @@ public class Token<T extends TokenIdentifier> implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Encode this token as a url safe string.
|
||||
* Encode this token as a url safe string
|
||||
* @return the encoded string
|
||||
* @throws IOException
|
||||
*/
|
||||
|
@ -407,7 +375,7 @@ public class Token<T extends TokenIdentifier> implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Renew this delegation token.
|
||||
* Renew this delegation token
|
||||
* @return the new expiration time
|
||||
* @throws IOException
|
||||
* @throws InterruptedException
|
||||
|
@ -418,7 +386,7 @@ public class Token<T extends TokenIdentifier> implements Writable {
|
|||
}
|
||||
|
||||
/**
|
||||
* Cancel this delegation token.
|
||||
* Cancel this delegation token
|
||||
* @throws IOException
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
|
|
|
@ -1,114 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.tools;
|
||||
|
||||
import java.io.PrintStream;
|
||||
|
||||
import org.apache.hadoop.conf.Configured;
|
||||
import org.apache.hadoop.util.Tool;
|
||||
|
||||
/**
|
||||
* This program is a CLI utility base class utilizing hadoop Tool class.
|
||||
*/
|
||||
public abstract class CommandShell extends Configured implements Tool {
|
||||
|
||||
private PrintStream out = System.out;
|
||||
private PrintStream err = System.err;
|
||||
|
||||
/** The subcommand instance for this shell command, if any. */
|
||||
private SubCommand subcommand = null;
|
||||
|
||||
/**
|
||||
* Return usage string for the command including any summary of subcommands.
|
||||
*/
|
||||
public abstract String getCommandUsage();
|
||||
|
||||
public void setSubCommand(SubCommand cmd) {
|
||||
subcommand = cmd;
|
||||
}
|
||||
|
||||
public void setOut(PrintStream p) {
|
||||
out = p;
|
||||
}
|
||||
|
||||
public PrintStream getOut() {
|
||||
return out;
|
||||
}
|
||||
|
||||
public void setErr(PrintStream p) {
|
||||
err = p;
|
||||
}
|
||||
|
||||
public PrintStream getErr() {
|
||||
return err;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int run(String[] args) throws Exception {
|
||||
int exitCode = 0;
|
||||
try {
|
||||
exitCode = init(args);
|
||||
if (exitCode != 0) {
|
||||
printShellUsage();
|
||||
return exitCode;
|
||||
}
|
||||
if (subcommand.validate()) {
|
||||
subcommand.execute();
|
||||
} else {
|
||||
printShellUsage();
|
||||
exitCode = 1;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
printShellUsage();
|
||||
e.printStackTrace(err);
|
||||
return 1;
|
||||
}
|
||||
return exitCode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the command line arguments and initialize subcommand instance.
|
||||
* @param args
|
||||
* @return 0 if the argument(s) were recognized, 1 otherwise
|
||||
*/
|
||||
protected abstract int init(String[] args) throws Exception;
|
||||
|
||||
private void printShellUsage() {
|
||||
if (subcommand != null) {
|
||||
out.println(subcommand.getUsage());
|
||||
} else {
|
||||
out.println(getCommandUsage());
|
||||
}
|
||||
out.flush();
|
||||
}
|
||||
|
||||
/**
|
||||
* Base class for any subcommands of this shell command.
|
||||
*/
|
||||
protected abstract class SubCommand {
|
||||
|
||||
public boolean validate() {
|
||||
return true;
|
||||
}
|
||||
|
||||
public abstract void execute() throws Exception;
|
||||
|
||||
public abstract String getUsage();
|
||||
}
|
||||
}
|
|
@ -38,17 +38,6 @@ message TokenProto {
|
|||
required string service = 4;
|
||||
}
|
||||
|
||||
message CredentialsKVProto {
|
||||
required string alias = 1;
|
||||
optional hadoop.common.TokenProto token = 2;
|
||||
optional bytes secret = 3;
|
||||
}
|
||||
|
||||
message CredentialsProto {
|
||||
repeated hadoop.common.CredentialsKVProto tokens = 1;
|
||||
repeated hadoop.common.CredentialsKVProto secrets = 2;
|
||||
}
|
||||
|
||||
message GetDelegationTokenRequestProto {
|
||||
required string renewer = 1;
|
||||
}
|
||||
|
|
|
@ -134,25 +134,6 @@ Change the ownership and permissions on many files at once.
|
|||
|
||||
Copy file or directories recursively. More information can be found at [Hadoop DistCp Guide](../../hadoop-distcp/DistCp.html).
|
||||
|
||||
### `dtutil`
|
||||
|
||||
Usage: `hadoop dtutil [-keytab` *keytab_file* `-principal` *principal_name* `]` *subcommand* `[-format (java|protobuf)] [-alias` *alias* `] [-renewer` *renewer* `]` *filename...*
|
||||
|
||||
Utility to fetch and manage hadoop delegation tokens inside credentials files. It is intended to replace the simpler command `fetchdt`. There are multiple subcommands, each with their own flags and options.
|
||||
|
||||
For every subcommand that writes out a file, the `-format` option will specify the internal format to use. `java` is the legacy format that matches `fetchdt`. The default is `protobuf`.
|
||||
|
||||
For every subcommand that connects to a service, convenience flags are provided to specify the kerberos principal name and keytab file to use for auth.
|
||||
|
||||
| SUBCOMMAND | Description |
|
||||
|:---- |:---- |
|
||||
| `print` <br/> `[-alias` *alias* `]` <br/> *filename* `[` *filename2* `...]` | Print out the fields in the tokens contained in *filename* (and *filename2* ...). <br/> If *alias* is specified, print only tokens matching *alias*. Otherwise, print all tokens. |
|
||||
| `get` *URL* <br/> `[-service` *scheme* `]` <br/> `[-format (java|protobuf)]` <br/> `[-alias` *alias* `]` <br/> `[-renewer` *renewer* `]` <br/> *filename* | Fetch a token from service at *URL* and place it in *filename*. <br/> *URL* is required and must immediately follow `get`.<br/> *URL* is the service URL, e.g. *hdfs://localhost:9000*. <br/> *alias* will overwrite the service field in the token. <br/> It is intended for hosts that have external and internal names, e.g. *firewall.com:14000*. <br/> *filename* should come last and is the name of the token file. <br/> It will be created if it does not exist. Otherwise, token(s) are added to existing file. <br/> The `-service` flag should only be used with a URL which starts with `http` or `https`. <br/> The following are equivalent: *hdfs://localhost:9000/* vs. *http://localhost:9000* `-service` *hdfs* |
|
||||
| `append` <br/> `[-format (java|protobuf)]` <br/> *filename* *filename2* `[` *filename3* `...]` | Append the contents of the first N filenames onto the last filename. <br/> When tokens with common service fields are present in multiple files, earlier files' tokens are overwritten. <br/> That is, tokens present in the last file are always preserved. |
|
||||
| `remove -alias` *alias* <br/> `[-format (java|protobuf)]` <br/> *filename* `[` *filename2* `...]` | From each file specified, remove the tokens matching *alias* and write out each file using specified format. <br/> *alias* must be specified. |
|
||||
| `cancel -alias` *alias* <br/> `[-format (java|protobuf)]` <br/> *filename* `[` *filename2* `...]` | Just like `remove`, except the tokens are also cancelled using the service specified in the token object. <br/> *alias* must be specified. |
|
||||
| `renew -alias` *alias* <br/> `[-format (java|protobuf)]` <br/> *filename* `[` *filename2* `...]` | For each file specified, renew the tokens matching *alias* and write out each file using specified format. <br/> *alias* must be specified. |
|
||||
|
||||
### `fs`
|
||||
|
||||
This command is documented in the [File System Shell Guide](./FileSystemShell.html). It is a synonym for `hdfs dfs` when HDFS is in use.
|
||||
|
|
|
@ -1,41 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.security.token;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.io.Text;
|
||||
import org.apache.hadoop.security.Credentials;
|
||||
import org.apache.hadoop.security.token.Token;
|
||||
import org.apache.hadoop.security.token.DtFetcher;
|
||||
|
||||
public class TestDtFetcher implements DtFetcher {
|
||||
public Text getServiceName() {
|
||||
return TestDtUtilShell.SERVICE_GET;
|
||||
}
|
||||
|
||||
public boolean isTokenRequired() {
|
||||
return true;
|
||||
}
|
||||
|
||||
public Token<?> addDelegationTokens(Configuration conf,
|
||||
Credentials creds, String renewer, String url) throws Exception {
|
||||
creds.addToken(TestDtUtilShell.MOCK_TOKEN.getService(),
|
||||
TestDtUtilShell.MOCK_TOKEN);
|
||||
return TestDtUtilShell.MOCK_TOKEN;
|
||||
}
|
||||
}
|
|
@ -1,264 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.security.token;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.io.Text;
|
||||
import org.apache.hadoop.security.Credentials;
|
||||
import org.apache.hadoop.security.token.Token;
|
||||
import org.apache.hadoop.security.token.TokenIdentifier;
|
||||
import org.apache.hadoop.security.token.DtFetcher;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import org.mockito.Mockito;
|
||||
|
||||
public class TestDtUtilShell {
|
||||
private static byte[] IDENTIFIER = {
|
||||
0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72};
|
||||
private static byte[] PASSWORD = {
|
||||
0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64};
|
||||
private static Text KIND = new Text("testTokenKind");
|
||||
private static Text SERVICE = new Text("testTokenService");
|
||||
private static Text SERVICE2 = new Text("ecivreSnekoTtset");
|
||||
private static Configuration defaultConf = new Configuration();
|
||||
private static FileSystem localFs = null;
|
||||
private final String alias = "proxy_ip:1234";
|
||||
private final String renewer = "yarn";
|
||||
private final String getUrl = SERVICE_GET.toString() + "://localhost:9000/";
|
||||
private final String getUrl2 = "http://localhost:9000/";
|
||||
public static Text SERVICE_GET = new Text("testTokenServiceGet");
|
||||
public static Text KIND_GET = new Text("testTokenKindGet");
|
||||
public static Token<?> MOCK_TOKEN =
|
||||
new Token(IDENTIFIER, PASSWORD, KIND_GET, SERVICE_GET);
|
||||
static {
|
||||
try {
|
||||
defaultConf.set("fs.defaultFS", "file:///");
|
||||
localFs = FileSystem.getLocal(defaultConf);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("init failure", e);
|
||||
}
|
||||
}
|
||||
|
||||
private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
|
||||
private final Path workDir = new Path(
|
||||
System.getProperty("test.build.data", "/tmp"), "TestDtUtilShell");
|
||||
private final Path tokenFile = new Path(workDir, "testPrintTokenFile");
|
||||
private final Path tokenFile2 = new Path(workDir, "testPrintTokenFile2");
|
||||
private final Path tokenLegacyFile = new Path(workDir, "testPrintTokenFile3");
|
||||
private final Path tokenFileGet = new Path(workDir, "testGetTokenFile");
|
||||
private final String tokenFilename = tokenFile.toString();
|
||||
private final String tokenFilename2 = tokenFile2.toString();
|
||||
private final String tokenFilenameGet = tokenFileGet.toString();
|
||||
private String[] args = null;
|
||||
private DtUtilShell dt = null;
|
||||
private int rc = 0;
|
||||
|
||||
@Before
|
||||
public void setup() throws Exception {
|
||||
localFs.mkdirs(localFs.makeQualified(workDir));
|
||||
makeTokenFile(tokenFile, false, null);
|
||||
makeTokenFile(tokenFile2, false, SERVICE2);
|
||||
makeTokenFile(tokenLegacyFile, true, null);
|
||||
dt = new DtUtilShell();
|
||||
dt.setConf(new Configuration());
|
||||
dt.setOut(new PrintStream(outContent));
|
||||
outContent.reset();
|
||||
rc = 0;
|
||||
}
|
||||
|
||||
@After
|
||||
public void teardown() throws Exception {
|
||||
localFs.delete(localFs.makeQualified(workDir), true);
|
||||
}
|
||||
|
||||
public void makeTokenFile(Path tokenPath, boolean legacy, Text service)
|
||||
throws IOException {
|
||||
if (service == null) {
|
||||
service = SERVICE;
|
||||
}
|
||||
Credentials creds = new Credentials();
|
||||
Token<? extends TokenIdentifier> tok = (Token<? extends TokenIdentifier>)
|
||||
new Token(IDENTIFIER, PASSWORD, KIND, service);
|
||||
creds.addToken(tok.getService(), tok);
|
||||
if (legacy) {
|
||||
creds.writeLegacyTokenStorageLocalFile(new File(tokenPath.toString()));
|
||||
} else {
|
||||
creds.writeTokenStorageFile(tokenPath, defaultConf);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPrint() throws Exception {
|
||||
args = new String[] {"print", tokenFilename};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test simple print exit code", 0, rc);
|
||||
assertTrue("test simple print output kind:\n" + outContent.toString(),
|
||||
outContent.toString().contains(KIND.toString()));
|
||||
assertTrue("test simple print output service:\n" + outContent.toString(),
|
||||
outContent.toString().contains(SERVICE.toString()));
|
||||
|
||||
outContent.reset();
|
||||
args = new String[] {"print", tokenLegacyFile.toString()};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test legacy print exit code", 0, rc);
|
||||
assertTrue("test simple print output kind:\n" + outContent.toString(),
|
||||
outContent.toString().contains(KIND.toString()));
|
||||
assertTrue("test simple print output service:\n" + outContent.toString(),
|
||||
outContent.toString().contains(SERVICE.toString()));
|
||||
|
||||
outContent.reset();
|
||||
args = new String[] {
|
||||
"print", "-alias", SERVICE.toString(), tokenFilename};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test alias print exit code", 0, rc);
|
||||
assertTrue("test simple print output kind:\n" + outContent.toString(),
|
||||
outContent.toString().contains(KIND.toString()));
|
||||
assertTrue("test simple print output service:\n" + outContent.toString(),
|
||||
outContent.toString().contains(SERVICE.toString()));
|
||||
|
||||
outContent.reset();
|
||||
args = new String[] {
|
||||
"print", "-alias", "not-a-serivce", tokenFilename};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test no alias print exit code", 0, rc);
|
||||
assertFalse("test no alias print output kind:\n" + outContent.toString(),
|
||||
outContent.toString().contains(KIND.toString()));
|
||||
assertFalse("test no alias print output service:\n" + outContent.toString(),
|
||||
outContent.toString().contains(SERVICE.toString()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppend() throws Exception {
|
||||
args = new String[] {"append", tokenFilename, tokenFilename2};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test simple append exit code", 0, rc);
|
||||
args = new String[] {"print", tokenFilename2};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test simple append print exit code", 0, rc);
|
||||
assertTrue("test simple append output kind:\n" + outContent.toString(),
|
||||
outContent.toString().contains(KIND.toString()));
|
||||
assertTrue("test simple append output service:\n" + outContent.toString(),
|
||||
outContent.toString().contains(SERVICE.toString()));
|
||||
assertTrue("test simple append output service:\n" + outContent.toString(),
|
||||
outContent.toString().contains(SERVICE2.toString()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRemove() throws Exception {
|
||||
args = new String[] {"remove", "-alias", SERVICE.toString(), tokenFilename};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test simple remove exit code", 0, rc);
|
||||
args = new String[] {"print", tokenFilename};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test simple remove print exit code", 0, rc);
|
||||
assertFalse("test simple remove output kind:\n" + outContent.toString(),
|
||||
outContent.toString().contains(KIND.toString()));
|
||||
assertFalse("test simple remove output service:\n" + outContent.toString(),
|
||||
outContent.toString().contains(SERVICE.toString()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGet() throws Exception {
|
||||
args = new String[] {"get", getUrl, tokenFilenameGet};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test mocked get exit code", 0, rc);
|
||||
args = new String[] {"print", tokenFilenameGet};
|
||||
rc = dt.run(args);
|
||||
String oc = outContent.toString();
|
||||
assertEquals("test print after get exit code", 0, rc);
|
||||
assertTrue("test print after get output kind:\n" + oc,
|
||||
oc.contains(KIND_GET.toString()));
|
||||
assertTrue("test print after get output service:\n" + oc,
|
||||
oc.contains(SERVICE_GET.toString()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetWithServiceFlag() throws Exception {
|
||||
args = new String[] {"get", getUrl2, "-service", SERVICE_GET.toString(),
|
||||
tokenFilenameGet};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test mocked get with service flag exit code", 0, rc);
|
||||
args = new String[] {"print", tokenFilenameGet};
|
||||
rc = dt.run(args);
|
||||
String oc = outContent.toString();
|
||||
assertEquals("test print after get with service flag exit code", 0, rc);
|
||||
assertTrue("test print after get with service flag output kind:\n" + oc,
|
||||
oc.contains(KIND_GET.toString()));
|
||||
assertTrue("test print after get with service flag output service:\n" + oc,
|
||||
oc.contains(SERVICE_GET.toString()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetWithAliasFlag() throws Exception {
|
||||
args = new String[] {"get", getUrl, "-alias", alias, tokenFilenameGet};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test mocked get with alias flag exit code", 0, rc);
|
||||
args = new String[] {"print", tokenFilenameGet};
|
||||
rc = dt.run(args);
|
||||
String oc = outContent.toString();
|
||||
assertEquals("test print after get with alias flag exit code", 0, rc);
|
||||
assertTrue("test print after get with alias flag output kind:\n" + oc,
|
||||
oc.contains(KIND_GET.toString()));
|
||||
assertTrue("test print after get with alias flag output alias:\n" + oc,
|
||||
oc.contains(alias));
|
||||
assertFalse("test print after get with alias flag output old service:\n" +
|
||||
oc, oc.contains(SERVICE_GET.toString()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFormatJavaFlag() throws Exception {
|
||||
args = new String[] {"get", getUrl, "-format", "java", tokenFilenameGet};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test mocked get with java format flag exit code", 0, rc);
|
||||
Credentials creds = new Credentials();
|
||||
Credentials spyCreds = Mockito.spy(creds);
|
||||
DataInputStream in = new DataInputStream(
|
||||
new FileInputStream(tokenFilenameGet));
|
||||
spyCreds.readTokenStorageStream(in);
|
||||
Mockito.verify(spyCreds).readFields(in);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFormatProtoFlag() throws Exception {
|
||||
args = new String[] {
|
||||
"get", getUrl, "-format", "protobuf", tokenFilenameGet};
|
||||
rc = dt.run(args);
|
||||
assertEquals("test mocked get with protobuf format flag exit code", 0, rc);
|
||||
Credentials creds = new Credentials();
|
||||
Credentials spyCreds = Mockito.spy(creds);
|
||||
DataInputStream in = new DataInputStream(
|
||||
new FileInputStream(tokenFilenameGet));
|
||||
spyCreds.readTokenStorageStream(in);
|
||||
Mockito.verify(spyCreds).readProtos(in);
|
||||
}
|
||||
}
|
|
@ -1,128 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.tools;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.PrintStream;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.tools.CommandShell;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestCommandShell {
|
||||
|
||||
public class Example extends CommandShell {
|
||||
public static final String EXAMPLE = "example";
|
||||
public static final String HELLO = "hello";
|
||||
public static final String HELLO_MSG = "hello is running";
|
||||
public static final String GOODBYE = "goodbye";
|
||||
public static final String GOODBYE_MSG = "goodbye is running";
|
||||
|
||||
public String[] savedArgs = null;
|
||||
|
||||
@Override
|
||||
protected int init(String[] args) throws Exception {
|
||||
String command = args[0];
|
||||
if (command.equals(HELLO)) {
|
||||
setSubCommand(new Hello());
|
||||
} else if (command.equals(GOODBYE)) {
|
||||
setSubCommand(new Goodbye());
|
||||
} else{
|
||||
return 1;
|
||||
}
|
||||
savedArgs = args;
|
||||
return 0;
|
||||
}
|
||||
|
||||
public String getCommandUsage() {
|
||||
return EXAMPLE;
|
||||
}
|
||||
|
||||
public class Hello extends SubCommand {
|
||||
public static final String HELLO_USAGE = EXAMPLE + " hello";
|
||||
@Override
|
||||
public boolean validate() {
|
||||
return savedArgs.length == 1;
|
||||
}
|
||||
@Override
|
||||
public void execute() throws Exception {
|
||||
System.out.println(HELLO_MSG);
|
||||
}
|
||||
@Override
|
||||
public String getUsage() {
|
||||
return HELLO_USAGE;
|
||||
}
|
||||
}
|
||||
|
||||
public class Goodbye extends SubCommand {
|
||||
public static final String GOODBYE_USAGE = EXAMPLE + " goodbye";
|
||||
@Override
|
||||
public void execute() throws Exception {
|
||||
System.out.println(GOODBYE_MSG);
|
||||
}
|
||||
@Override
|
||||
public String getUsage() {
|
||||
return GOODBYE_USAGE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
|
||||
|
||||
private String outMsg(String message) {
|
||||
return "OUT:\n" + outContent.toString() + "\n" + message;
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setup() throws Exception {
|
||||
System.setOut(new PrintStream(outContent));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCommandShellExample() throws Exception {
|
||||
Example ex = new Example();
|
||||
ex.setConf(new Configuration());
|
||||
int rc = 0;
|
||||
|
||||
outContent.reset();
|
||||
String[] args1 = {"hello"};
|
||||
rc = ex.run(args1);
|
||||
assertEquals(outMsg("test exit code - normal hello"), 0, rc);
|
||||
assertTrue(outMsg("test normal hello message"),
|
||||
outContent.toString().contains(Example.HELLO_MSG));
|
||||
|
||||
outContent.reset();
|
||||
String[] args2 = {"hello", "x"};
|
||||
rc = ex.run(args2);
|
||||
assertEquals(outMsg("test exit code - bad hello"), 1, rc);
|
||||
assertTrue(outMsg("test bad hello message"),
|
||||
outContent.toString().contains(Example.Hello.HELLO_USAGE));
|
||||
|
||||
outContent.reset();
|
||||
String[] args3 = {"goodbye"};
|
||||
rc = ex.run(args3);
|
||||
assertEquals(outMsg("test exit code - normal goodbye"), 0, rc);
|
||||
assertTrue(outMsg("test normal goodbye message"),
|
||||
outContent.toString().contains(Example.GOODBYE_MSG));
|
||||
}
|
||||
}
|
|
@ -1,14 +0,0 @@
|
|||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
org.apache.hadoop.security.token.TestDtFetcher
|
|
@ -1,82 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hdfs;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||
import org.apache.hadoop.io.Text;
|
||||
import org.apache.hadoop.security.Credentials;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.token.DtFetcher;
|
||||
import org.apache.hadoop.security.token.Token;
|
||||
|
||||
|
||||
/**
|
||||
* DtFetcher is an interface which permits the abstraction and separation of
|
||||
* delegation token fetch implementaions across different packages and
|
||||
* compilation units. Resolution of fetcher impl will be done at runtime.
|
||||
*/
|
||||
public class HdfsDtFetcher implements DtFetcher {
|
||||
private static final Log LOG = LogFactory.getLog(HdfsDtFetcher.class);
|
||||
|
||||
private static final String SERVICE_NAME = HdfsConstants.HDFS_URI_SCHEME;
|
||||
|
||||
private static final String FETCH_FAILED = "Fetch of delegation token failed";
|
||||
|
||||
/**
|
||||
* Returns the service name for HDFS, which is also a valid URL prefix.
|
||||
*/
|
||||
public Text getServiceName() {
|
||||
return new Text(SERVICE_NAME);
|
||||
}
|
||||
|
||||
public boolean isTokenRequired() {
|
||||
return UserGroupInformation.isSecurityEnabled();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns Token object via FileSystem, null if bad argument.
|
||||
* @param conf - a Configuration object used with FileSystem.get()
|
||||
* @param creds - a Credentials object to which token(s) will be added
|
||||
* @param renewer - the renewer to send with the token request
|
||||
* @param url - the URL to which the request is sent
|
||||
* @return a Token, or null if fetch fails.
|
||||
*/
|
||||
public Token<?> addDelegationTokens(Configuration conf, Credentials creds,
|
||||
String renewer, String url) throws Exception {
|
||||
if (!url.startsWith(getServiceName().toString())) {
|
||||
url = getServiceName().toString() + "://" + url;
|
||||
}
|
||||
FileSystem fs = FileSystem.get(URI.create(url), conf);
|
||||
Token<?> token = fs.getDelegationToken(renewer);
|
||||
if (token == null) {
|
||||
LOG.error(FETCH_FAILED);
|
||||
throw new IOException(FETCH_FAILED);
|
||||
}
|
||||
creds.addToken(token.getService(), token);
|
||||
return token;
|
||||
}
|
||||
}
|
|
@ -1,39 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hdfs;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
|
||||
import org.apache.hadoop.io.Text;
|
||||
|
||||
/**
|
||||
* DtFetcher for SWebHdfsFileSystem using the base class HdfsDtFetcher impl.
|
||||
*/
|
||||
public class SWebHdfsDtFetcher extends HdfsDtFetcher {
|
||||
private static final Log LOG = LogFactory.getLog(SWebHdfsDtFetcher.class);
|
||||
|
||||
private static final String SERVICE_NAME = WebHdfsConstants.SWEBHDFS_SCHEME;
|
||||
|
||||
@Override
|
||||
public Text getServiceName() {
|
||||
return new Text(SERVICE_NAME);
|
||||
}
|
||||
}
|
|
@ -1,39 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hdfs;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
|
||||
import org.apache.hadoop.io.Text;
|
||||
|
||||
/**
|
||||
* DtFetcher for WebHdfsFileSystem using the base class HdfsDtFetcher impl.
|
||||
*/
|
||||
public class WebHdfsDtFetcher extends HdfsDtFetcher {
|
||||
private static final Log LOG = LogFactory.getLog(WebHdfsDtFetcher.class);
|
||||
|
||||
private static final String SERVICE_NAME = WebHdfsConstants.WEBHDFS_SCHEME;
|
||||
|
||||
@Override
|
||||
public Text getServiceName() {
|
||||
return new Text(SERVICE_NAME);
|
||||
}
|
||||
}
|
|
@ -179,8 +179,7 @@ public class DelegationTokenFetcher {
|
|||
if (null != token) {
|
||||
Credentials cred = new Credentials();
|
||||
cred.addToken(token.getService(), token);
|
||||
// dtutil is replacing this tool; preserve legacy functionality
|
||||
cred.writeLegacyTokenStorageFile(tokenFile, conf);
|
||||
cred.writeTokenStorageFile(tokenFile, conf);
|
||||
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Fetched token " + fs.getUri() + " for " +
|
||||
|
|
|
@ -1,18 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
org.apache.hadoop.hdfs.HdfsDtFetcher
|
||||
org.apache.hadoop.hdfs.WebHdfsDtFetcher
|
||||
org.apache.hadoop.hdfs.SWebHdfsDtFetcher
|
Loading…
Reference in New Issue