HADOOP-6472. add tokenCache option to GenericOptionsParser for passing
file with secret keys to a map reduce job. (boryas) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@895801 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
e683a0d221
commit
3cb2e3112b
|
@ -77,6 +77,9 @@ Trunk (unreleased changes)
|
|||
HADOOP-6435. Make RPC.waitForProxy with timeout public. (Steve Loughran
|
||||
via tomwhite)
|
||||
|
||||
HADOOP-6472. add tokenCache option to GenericOptionsParser for passing
|
||||
file with secret keys to a map reduce job. (boryas)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
|
|
@ -234,6 +234,12 @@ public class GenericOptionsParser {
|
|||
" on the compute machines.")
|
||||
.create("archives");
|
||||
|
||||
// file with security tokens
|
||||
Option tokensFile = OptionBuilder.withArgName("tokensFile")
|
||||
.hasArg()
|
||||
.withDescription("name of the file with the tokens")
|
||||
.create("tokenCacheFile");
|
||||
|
||||
opts.addOption(fs);
|
||||
opts.addOption(jt);
|
||||
opts.addOption(oconf);
|
||||
|
@ -241,6 +247,7 @@ public class GenericOptionsParser {
|
|||
opts.addOption(libjars);
|
||||
opts.addOption(files);
|
||||
opts.addOption(archives);
|
||||
opts.addOption(tokensFile);
|
||||
|
||||
return opts;
|
||||
}
|
||||
|
@ -295,6 +302,19 @@ public class GenericOptionsParser {
|
|||
}
|
||||
}
|
||||
conf.setBoolean("mapred.used.genericoptionsparser", true);
|
||||
|
||||
// tokensFile
|
||||
if(line.hasOption("tokenCacheFile")) {
|
||||
String fileName = line.getOptionValue("tokenCacheFile");
|
||||
// check if the local file exists
|
||||
FileSystem localFs = FileSystem.getLocal(conf);
|
||||
Path p = new Path(fileName);
|
||||
if (!localFs.exists(p)) {
|
||||
throw new FileNotFoundException("File "+fileName+" does not exist.");
|
||||
}
|
||||
LOG.debug("setting conf tokensFile: " + fileName);
|
||||
conf.set("tokenCacheFile", localFs.makeQualified(p).toString());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -19,22 +19,23 @@ package org.apache.hadoop.util;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
public class TestGenericOptionsParser extends TestCase {
|
||||
private static File testDir =
|
||||
new File(System.getProperty("test.build.data", "/tmp"), "generic");
|
||||
File testDir;
|
||||
Configuration conf;
|
||||
FileSystem localFs;
|
||||
|
||||
|
||||
public void testFilesOption() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
File tmpFile = new File(testDir, "tmpfile");
|
||||
FileSystem localFs = FileSystem.getLocal(conf);
|
||||
Path tmpPath = new Path(tmpFile.toString());
|
||||
localFs.create(tmpPath);
|
||||
String[] args = new String[2];
|
||||
|
@ -74,7 +75,62 @@ public class TestGenericOptionsParser extends TestCase {
|
|||
th instanceof FileNotFoundException);
|
||||
files = conf2.get("tmpfiles");
|
||||
assertNull("files is not null", files);
|
||||
testDir.delete();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setUp() throws Exception {
|
||||
super.setUp();
|
||||
conf = new Configuration();
|
||||
localFs = FileSystem.getLocal(conf);
|
||||
testDir = new File(System.getProperty("test.build.data", "/tmp"), "generic");
|
||||
if(testDir.exists())
|
||||
localFs.delete(new Path(testDir.toString()), true);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void tearDown() throws Exception {
|
||||
super.tearDown();
|
||||
if(testDir.exists()) {
|
||||
localFs.delete(new Path(testDir.toString()), true);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* testing -fileCache option
|
||||
* @throws IOException
|
||||
*/
|
||||
public void testTokenCacheOption() throws IOException {
|
||||
FileSystem localFs = FileSystem.getLocal(conf);
|
||||
|
||||
File tmpFile = new File(testDir, "tokenCacheFile");
|
||||
if(tmpFile.exists()) {
|
||||
tmpFile.delete();
|
||||
}
|
||||
String[] args = new String[2];
|
||||
// pass a files option
|
||||
args[0] = "-tokenCacheFile";
|
||||
args[1] = tmpFile.toString();
|
||||
|
||||
// test non existing file
|
||||
Throwable th = null;
|
||||
try {
|
||||
new GenericOptionsParser(conf, args);
|
||||
} catch (Exception e) {
|
||||
th = e;
|
||||
}
|
||||
assertNotNull(th);
|
||||
assertTrue("FileNotFoundException is not thrown",
|
||||
th instanceof FileNotFoundException);
|
||||
|
||||
// create file
|
||||
Path tmpPath = new Path(tmpFile.toString());
|
||||
localFs.create(tmpPath);
|
||||
new GenericOptionsParser(conf, args);
|
||||
String fileName = conf.get("tokenCacheFile");
|
||||
assertNotNull("files is null", fileName);
|
||||
assertEquals("files option does not match",
|
||||
localFs.makeQualified(tmpPath).toString(), fileName);
|
||||
|
||||
localFs.delete(new Path(testDir.getAbsolutePath()), true);
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue