HADOOP-6472. add tokenCache option to GenericOptionsParser for passing

file with secret keys to a map reduce job. (boryas)


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@895801 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Boris Shkolnik 2010-01-04 22:14:34 +00:00
parent e683a0d221
commit 3cb2e3112b
3 changed files with 86 additions and 7 deletions

View File

@ -76,6 +76,9 @@ Trunk (unreleased changes)
HADOOP-6435. Make RPC.waitForProxy with timeout public. (Steve Loughran HADOOP-6435. Make RPC.waitForProxy with timeout public. (Steve Loughran
via tomwhite) via tomwhite)
HADOOP-6472. add tokenCache option to GenericOptionsParser for passing
file with secret keys to a map reduce job. (boryas)
OPTIMIZATIONS OPTIMIZATIONS

View File

@ -233,6 +233,12 @@ public class GenericOptionsParser {
.withDescription("comma separated archives to be unarchived" + .withDescription("comma separated archives to be unarchived" +
" on the compute machines.") " on the compute machines.")
.create("archives"); .create("archives");
// file with security tokens
Option tokensFile = OptionBuilder.withArgName("tokensFile")
.hasArg()
.withDescription("name of the file with the tokens")
.create("tokenCacheFile");
opts.addOption(fs); opts.addOption(fs);
opts.addOption(jt); opts.addOption(jt);
@ -241,6 +247,7 @@ public class GenericOptionsParser {
opts.addOption(libjars); opts.addOption(libjars);
opts.addOption(files); opts.addOption(files);
opts.addOption(archives); opts.addOption(archives);
opts.addOption(tokensFile);
return opts; return opts;
} }
@ -295,6 +302,19 @@ public class GenericOptionsParser {
} }
} }
conf.setBoolean("mapred.used.genericoptionsparser", true); conf.setBoolean("mapred.used.genericoptionsparser", true);
// tokensFile
if(line.hasOption("tokenCacheFile")) {
String fileName = line.getOptionValue("tokenCacheFile");
// check if the local file exists
FileSystem localFs = FileSystem.getLocal(conf);
Path p = new Path(fileName);
if (!localFs.exists(p)) {
throw new FileNotFoundException("File "+fileName+" does not exist.");
}
LOG.debug("setting conf tokensFile: " + fileName);
conf.set("tokenCacheFile", localFs.makeQualified(p).toString());
}
} }
/** /**

View File

@ -19,22 +19,23 @@ package org.apache.hadoop.util;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI; import java.net.URI;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import junit.framework.TestCase;
public class TestGenericOptionsParser extends TestCase { public class TestGenericOptionsParser extends TestCase {
private static File testDir = File testDir;
new File(System.getProperty("test.build.data", "/tmp"), "generic"); Configuration conf;
FileSystem localFs;
public void testFilesOption() throws Exception { public void testFilesOption() throws Exception {
Configuration conf = new Configuration();
File tmpFile = new File(testDir, "tmpfile"); File tmpFile = new File(testDir, "tmpfile");
FileSystem localFs = FileSystem.getLocal(conf);
Path tmpPath = new Path(tmpFile.toString()); Path tmpPath = new Path(tmpFile.toString());
localFs.create(tmpPath); localFs.create(tmpPath);
String[] args = new String[2]; String[] args = new String[2];
@ -74,7 +75,62 @@ public class TestGenericOptionsParser extends TestCase {
th instanceof FileNotFoundException); th instanceof FileNotFoundException);
files = conf2.get("tmpfiles"); files = conf2.get("tmpfiles");
assertNull("files is not null", files); assertNull("files is not null", files);
testDir.delete(); }
@Override
protected void setUp() throws Exception {
super.setUp();
conf = new Configuration();
localFs = FileSystem.getLocal(conf);
testDir = new File(System.getProperty("test.build.data", "/tmp"), "generic");
if(testDir.exists())
localFs.delete(new Path(testDir.toString()), true);
} }
@Override
protected void tearDown() throws Exception {
super.tearDown();
if(testDir.exists()) {
localFs.delete(new Path(testDir.toString()), true);
}
}
/**
* testing -fileCache option
* @throws IOException
*/
public void testTokenCacheOption() throws IOException {
FileSystem localFs = FileSystem.getLocal(conf);
File tmpFile = new File(testDir, "tokenCacheFile");
if(tmpFile.exists()) {
tmpFile.delete();
}
String[] args = new String[2];
// pass a files option
args[0] = "-tokenCacheFile";
args[1] = tmpFile.toString();
// test non existing file
Throwable th = null;
try {
new GenericOptionsParser(conf, args);
} catch (Exception e) {
th = e;
}
assertNotNull(th);
assertTrue("FileNotFoundException is not thrown",
th instanceof FileNotFoundException);
// create file
Path tmpPath = new Path(tmpFile.toString());
localFs.create(tmpPath);
new GenericOptionsParser(conf, args);
String fileName = conf.get("tokenCacheFile");
assertNotNull("files is null", fileName);
assertEquals("files option does not match",
localFs.makeQualified(tmpPath).toString(), fileName);
localFs.delete(new Path(testDir.getAbsolutePath()), true);
}
} }