Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11432. Federation : Support fully qualified path for Quota/Snapshot/cacheadmin/cryptoadmin commands. Contributed by Brahma Reddy Battula. 

(cherry picked from commit dcd03df9f9)
This commit is contained in:
Brahma Reddy Battula 2017-03-01 10:45:56 +05:30
parent 265ddb20c3
commit 31bec2c16b
7 changed files with 136 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
View File

@ -100,7 +100,17 @@ abstract public class Command extends Configured {
* @throws IOException if any error occurs
*/
abstract protected void run(Path path) throws IOException;
/**
* Execute the command on the input path data. Commands can override to make
* use of the resolved filesystem.
* @param pathData The input path with resolved filesystem
* @throws IOException
*/
protected void run(PathData pathData) throws IOException {
run(pathData.path);
}
/**
* For each source path, execute the command
*
@ -112,7 +122,7 @@ abstract public class Command extends Configured {
try {
PathData[] srcs = PathData.expandAsGlob(src, getConf());
for (PathData s : srcs) {
run(s.path);
run(s);
}
} catch (IOException e) {
exitCode = -1;

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
View File

@ -185,8 +185,9 @@ public class CacheAdmin extends Configured implements Tool {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
}
DistributedFileSystem dfs = AdminHelper.getDFS(conf);
DistributedFileSystem dfs =
AdminHelper.getDFS(new Path(path).toUri(), conf);
CacheDirectiveInfo directive = builder.build();
EnumSet<CacheFlag> flags = EnumSet.noneOf(CacheFlag.class);
if (force) {
@ -405,7 +406,8 @@ public class CacheAdmin extends Configured implements Tool {
}
int exitCode = 0;
try {
DistributedFileSystem dfs = AdminHelper.getDFS(conf);
DistributedFileSystem dfs =
AdminHelper.getDFS(new Path(path).toUri(), conf);
RemoteIterator<CacheDirectiveEntry> iter =
dfs.listCacheDirectives(
new CacheDirectiveInfo.Builder().

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
View File

@ -135,12 +135,12 @@ public class CryptoAdmin extends Configured implements Tool {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
}
HdfsAdmin admin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
Path p = new Path(path);
HdfsAdmin admin = new HdfsAdmin(p.toUri(), conf);
EnumSet<CreateEncryptionZoneFlag> flags =
EnumSet.of(CreateEncryptionZoneFlag.PROVISION_TRASH);
try {
admin.createEncryptionZone(new Path(path), keyName, flags);
admin.createEncryptionZone(p, keyName, flags);
System.out.println("Added encryption zone " + path);
} catch (IOException e) {
System.err.println(prettifyException(e));
@ -222,12 +222,12 @@ public class CryptoAdmin extends Configured implements Tool {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
}
Path p = new Path(path);
final HdfsAdmin admin =
new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
new HdfsAdmin(p.toUri(), conf);
try {
final FileEncryptionInfo fei =
admin.getFileEncryptionInfo(new Path(path));
admin.getFileEncryptionInfo(p);
if (fei == null) {
System.out.println("No FileEncryptionInfo found for path " + path);
return 2;
@ -269,10 +269,10 @@ public class CryptoAdmin extends Configured implements Tool {
System.err.println("Can't understand argument: " + args.get(0));
return 1;
}
HdfsAdmin admin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
Path p = new Path(path);
HdfsAdmin admin = new HdfsAdmin(p.toUri(), conf);
try {
admin.provisionEncryptionZoneTrash(new Path(path));
admin.provisionEncryptionZoneTrash(p);
System.out.println("Created a trash directory for " + path);
} catch (IOException ioe) {
System.err.println(prettifyException(ioe));

57
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
View File

@ -51,6 +51,7 @@ import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.Command;
import org.apache.hadoop.fs.shell.CommandFormat;
import org.apache.hadoop.fs.shell.PathData;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HAUtilClient;
@ -107,15 +108,21 @@ public class DFSAdmin extends FsShell {
* An abstract class for the execution of a file system command
*/
abstract private static class DFSAdminCommand extends Command {
final DistributedFileSystem dfs;
protected DistributedFileSystem dfs;
/** Constructor */
public DFSAdminCommand(FileSystem fs) {
super(fs.getConf());
public DFSAdminCommand(Configuration conf) {
super(conf);
}
@Override
public void run(PathData pathData) throws IOException {
FileSystem fs = pathData.fs;
if (!(fs instanceof DistributedFileSystem)) {
throw new IllegalArgumentException("FileSystem " + fs.getUri() +
" is not an HDFS file system");
throw new IllegalArgumentException("FileSystem " + fs.getUri()
+ " is not an HDFS file system");
}
this.dfs = (DistributedFileSystem)fs;
this.dfs = (DistributedFileSystem) fs;
run(pathData.path);
}
}
@ -131,8 +138,8 @@ public class DFSAdmin extends FsShell {
"\t\tIt does not fault if the directory has no quota.";
/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
super(fs);
ClearQuotaCommand(String[] args, int pos, Configuration conf) {
super(conf);
CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
List<String> parameters = c.parse(args, pos);
this.args = parameters.toArray(new String[parameters.size()]);
@ -177,8 +184,8 @@ public class DFSAdmin extends FsShell {
private final long quota; // the quota to be set
/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
super(fs);
SetQuotaCommand(String[] args, int pos, Configuration conf) {
super(conf);
CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
List<String> parameters = c.parse(args, pos);
this.quota = Long.parseLong(parameters.remove(0));
@ -228,8 +235,8 @@ public class DFSAdmin extends FsShell {
private StorageType type;
/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
super(fs);
ClearSpaceQuotaCommand(String[] args, int pos, Configuration conf) {
super(conf);
CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
c.addOptionWithValue("storageType");
List<String> parameters = c.parse(args, pos);
@ -292,8 +299,8 @@ public class DFSAdmin extends FsShell {
private StorageType type;
/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
super(fs);
SetSpaceQuotaCommand(String[] args, int pos, Configuration conf) {
super(conf);
CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
List<String> parameters = c.parse(args, pos);
String str = parameters.remove(0).trim();
@ -702,10 +709,11 @@ public class DFSAdmin extends FsShell {
* @param argv List of of command line parameters.
* @exception IOException
*/
public void allowSnapshot(String[] argv) throws IOException {
DistributedFileSystem dfs = getDFS();
public void allowSnapshot(String[] argv) throws IOException {
Path p = new Path(argv[1]);
final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), getConf());
try {
dfs.allowSnapshot(new Path(argv[1]));
dfs.allowSnapshot(p);
} catch (SnapshotException e) {
throw new RemoteException(e.getClass().getName(), e.getMessage());
}
@ -718,10 +726,11 @@ public class DFSAdmin extends FsShell {
* @param argv List of of command line parameters.
* @exception IOException
*/
public void disallowSnapshot(String[] argv) throws IOException {
DistributedFileSystem dfs = getDFS();
public void disallowSnapshot(String[] argv) throws IOException {
Path p = new Path(argv[1]);
final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), getConf());
try {
dfs.disallowSnapshot(new Path(argv[1]));
dfs.disallowSnapshot(p);
} catch (SnapshotException e) {
throw new RemoteException(e.getClass().getName(), e.getMessage());
}
@ -1927,13 +1936,13 @@ public class DFSAdmin extends FsShell {
} else if ("-metasave".equals(cmd)) {
exitCode = metaSave(argv, i);
} else if (ClearQuotaCommand.matches(cmd)) {
exitCode = new ClearQuotaCommand(argv, i, getDFS()).runAll();
exitCode = new ClearQuotaCommand(argv, i, getConf()).runAll();
} else if (SetQuotaCommand.matches(cmd)) {
exitCode = new SetQuotaCommand(argv, i, getDFS()).runAll();
exitCode = new SetQuotaCommand(argv, i, getConf()).runAll();
} else if (ClearSpaceQuotaCommand.matches(cmd)) {
exitCode = new ClearSpaceQuotaCommand(argv, i, getDFS()).runAll();
exitCode = new ClearSpaceQuotaCommand(argv, i, getConf()).runAll();
} else if (SetSpaceQuotaCommand.matches(cmd)) {
exitCode = new SetSpaceQuotaCommand(argv, i, getDFS()).runAll();
exitCode = new SetSpaceQuotaCommand(argv, i, getConf()).runAll();
} else if ("-refreshServiceAcl".equals(cmd)) {
exitCode = refreshServiceAcl();
} else if ("-refreshUserToGroupsMappings".equals(cmd)) {

19
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
View File

@ -20,10 +20,12 @@ package org.apache.hadoop.hdfs.tools.snapshot;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.util.Tool;
@ -41,6 +43,19 @@ import org.apache.hadoop.util.ToolRunner;
*/
@InterfaceAudience.Private
public class SnapshotDiff extends Configured implements Tool {
/**
* Construct a SnapshotDiff object.
*/
public SnapshotDiff() {
this(new HdfsConfiguration());
}
/**
* Construct a SnapshotDiff object.
*/
public SnapshotDiff(Configuration conf) {
super(conf);
}
private static String getSnapshotName(String name) {
if (Path.CUR_DIR.equals(name)) { // current directory
return "";
@ -72,8 +87,8 @@ public class SnapshotDiff extends Configured implements Tool {
System.err.println("Usage: \n" + description);
return 1;
}
FileSystem fs = FileSystem.get(getConf());
FileSystem fs = FileSystem.get(new Path(argv[0]).toUri(), getConf());
if (! (fs instanceof DistributedFileSystem)) {
System.err.println(
"SnapshotDiff can only be used in DistributedFileSystem");

26
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
View File

@ -1238,6 +1238,32 @@ public class TestQuota {
-1);
}
/**
* Test to all the commands by passing the fully qualified path.
*/
@Test(timeout = 30000)
public void testQuotaCommandsWithURI() throws Exception {
DFSAdmin dfsAdmin = new DFSAdmin(conf);
final Path dir = new Path("/" + this.getClass().getSimpleName(),
GenericTestUtils.getMethodName());
assertTrue(dfs.mkdirs(dir));
/* set space quota */
testSetAndClearSpaceQuotaRegularInternal(
new String[] { "-setSpaceQuota", "1024",
dfs.getUri() + "/" + dir.toString() }, dir, 0, 1024);
/* clear space quota */
testSetAndClearSpaceQuotaRegularInternal(
new String[] { "-clrSpaceQuota", dfs.getUri() + "/" + dir.toString() },
dir, 0, -1);
runCommand(dfsAdmin, false, "-setQuota", "1000",
dfs.getUri() + "/" + dir.toString());
runCommand(dfsAdmin, false, "-clrQuota",
dfs.getUri() + "/" + dir.toString());
}
private void testSetAndClearSpaceQuotaRegularInternal(
final String[] args,
final Path dir,

34
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSnapshotCommands.java
View File

@ -23,6 +23,8 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@ -60,6 +62,7 @@ public class TestSnapshotCommands {
@Before
public void setUp() throws IOException {
fs.mkdirs(new Path("/sub1"));
fs.mkdirs(new Path("/Fully/QPath"));
fs.allowSnapshot(new Path("/sub1"));
fs.mkdirs(new Path("/sub1/sub1sub1"));
fs.mkdirs(new Path("/sub1/sub1sub2"));
@ -161,4 +164,35 @@ public class TestSnapshotCommands {
// now it can be deleted
DFSTestUtil.FsShellRun("-rmr /sub1", conf);
}
@Test (timeout=60000)
public void testSnapshotCommandsWithURI()throws Exception {
Configuration config = new HdfsConfiguration();
//fs.defaultFS should not be used, when path is fully qualified.
config.set("fs.defaultFS", "hdfs://127.0.0.1:1024");
String path = fs.getUri() + "/Fully/QPath";
DFSTestUtil.DFSAdminRun("-allowSnapshot " + path, 0,
"Allowing snaphot on " + path + " succeeded", config);
DFSTestUtil.FsShellRun("-createSnapshot " + path + " sn1", config);
// create file1
DFSTestUtil
.createFile(fs, new Path(fs.getUri() + "/Fully/QPath/File1"), 1024,
(short) 1, 100);
// create file2
DFSTestUtil
.createFile(fs, new Path(fs.getUri() + "/Fully/QPath/File2"), 1024,
(short) 1, 100);
DFSTestUtil.FsShellRun("-createSnapshot " + path + " sn2", config);
// verify the snapshotdiff using api and command line
SnapshotDiffReport report =
fs.getSnapshotDiffReport(new Path(path), "sn1", "sn2");
DFSTestUtil.toolRun(new SnapshotDiff(config), path + " sn1 sn2", 0,
report.toString());
DFSTestUtil.FsShellRun("-renameSnapshot " + path + " sn2 sn3", config);
DFSTestUtil.FsShellRun("-deleteSnapshot " + path + " sn1", config);
DFSTestUtil.FsShellRun("-deleteSnapshot " + path + " sn3", config);
DFSTestUtil.DFSAdminRun("-disallowSnapshot " + path, 0,
"Disallowing snaphot on " + path + " succeeded", config);
fs.delete(new Path("/Fully/QPath"), true);
}
}