Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14096. [SPS] : Add Support for Storage Policy Satisfier in ViewFs. Contributed by Ayush Saxena. 

(cherry picked from commit 788e7473a4)
This commit is contained in:
Surendra Singh Lilhore 2018-12-17 11:24:57 +05:30 committed by Uma Maheswara Rao G
parent 0512b27172
commit 3bacea2e5e
13 changed files with 96 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
View File

@ -1249,6 +1249,16 @@ public abstract class AbstractFileSystem implements PathCapabilities {
+ " doesn't support deleteSnapshot");
}
/**
* Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
*/
public void satisfyStoragePolicy(final Path path) throws IOException {
throw new UnsupportedOperationException(
getClass().getSimpleName() + " doesn't support satisfyStoragePolicy");
}
/**
* Set the storage policy for a given file or directory.
*

18
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
View File

@ -2777,6 +2777,24 @@ public class FileContext implements PathCapabilities {
}.resolve(this, absF);
}
/**
* Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
*/
public void satisfyStoragePolicy(final Path path)
throws IOException {
final Path absF = fixRelativePart(path);
new FSLinkResolver<Void>() {
@Override
public Void next(final AbstractFileSystem fs, final Path p)
throws IOException {
fs.satisfyStoragePolicy(path);
return null;
}
}.resolve(this, absF);
}
/**
* Set the storage policy for a given file or directory.
*

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
View File

@ -3115,6 +3115,16 @@ public abstract class FileSystem extends Configured
+ " doesn't support removeXAttr");
}
/**
* Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
*/
public void satisfyStoragePolicy(final Path path) throws IOException {
throw new UnsupportedOperationException(
getClass().getSimpleName() + " doesn't support setStoragePolicy");
}
/**
* Set the storage policy for a given file or directory.
*

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
View File

@ -649,6 +649,11 @@ public class FilterFileSystem extends FileSystem {
fs.removeXAttr(path, name);
}
@Override
public void satisfyStoragePolicy(Path src) throws IOException {
fs.satisfyStoragePolicy(src);
}
@Override
public void setStoragePolicy(Path src, String policyName)
throws IOException {

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
View File

@ -405,6 +405,11 @@ public abstract class FilterFs extends AbstractFileSystem {
myFs.deleteSnapshot(path, snapshotName);
}
@Override
public void satisfyStoragePolicy(final Path path) throws IOException {
myFs.satisfyStoragePolicy(path);
}
@Override
public void setStoragePolicy(Path path, String policyName)
throws IOException {

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
View File

@ -464,6 +464,11 @@ class ChRootedFileSystem extends FilterFileSystem {
return super.getStoragePolicy(fullPath(src));
}
@Override
public void satisfyStoragePolicy(Path src) throws IOException {
super.satisfyStoragePolicy(fullPath(src));
}
@Override
public void setStoragePolicy(Path src, String policyName) throws IOException {
super.setStoragePolicy(fullPath(src), policyName);

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
View File

@ -398,6 +398,11 @@ class ChRootedFs extends AbstractFileSystem {
myFs.deleteSnapshot(fullPath(snapshotDir), snapshotName);
}
@Override
public void satisfyStoragePolicy(final Path path) throws IOException {
myFs.satisfyStoragePolicy(path);
}
@Override
public void setStoragePolicy(Path path, String policyName)
throws IOException {

13
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -978,6 +978,13 @@ public class ViewFileSystem extends FileSystem {
res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
}
@Override
public void satisfyStoragePolicy(Path src) throws IOException {
InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(src), true);
res.targetFileSystem.satisfyStoragePolicy(res.remainingPath);
}
@Override
public void setStoragePolicy(Path src, String policyName) throws IOException {
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
@ -1625,6 +1632,12 @@ public class ViewFileSystem extends FileSystem {
throw new NotInMountpointException(f, "getQuotaUsage");
}
@Override
public void satisfyStoragePolicy(Path src) throws IOException {
checkPathIsSlash(src);
throw readOnlyMountTable("satisfyStoragePolicy", src);
}
@Override
public void setStoragePolicy(Path src, String policyName)
throws IOException {

12
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
View File

@ -811,6 +811,13 @@ public class ViewFs extends AbstractFileSystem {
res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
}
@Override
public void satisfyStoragePolicy(final Path path) throws IOException {
InodeTree.ResolveResult<AbstractFileSystem> res =
fsState.resolve(getUriPath(path), true);
res.targetFileSystem.satisfyStoragePolicy(res.remainingPath);
}
@Override
public void setStoragePolicy(final Path path, final String policyName)
throws IOException {
@ -1377,6 +1384,11 @@ public class ViewFs extends AbstractFileSystem {
throw readOnlyMountTable("deleteSnapshot", path);
}
@Override
public void satisfyStoragePolicy(final Path path) throws IOException {
throw readOnlyMountTable("satisfyStoragePolicy", path);
}
@Override
public void setStoragePolicy(Path path, String policyName)
throws IOException {

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
View File

@ -215,6 +215,8 @@ public class TestHarFileSystem {
public void access(Path path, FsAction mode) throws IOException;
void satisfyStoragePolicy(Path src) throws IOException;
public void setStoragePolicy(Path src, String policyName)
throws IOException;

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
View File

@ -981,6 +981,11 @@ abstract public class ViewFileSystemBaseTest {
fsView.unsetStoragePolicy(new Path("/internalDir"));
}
@Test(expected = AccessControlException.class)
public void testInternalSatisfyStoragePolicy() throws IOException {
fsView.satisfyStoragePolicy(new Path("/internalDir"));
}
@Test(expected = NotInMountpointException.class)
public void testInternalgetStoragePolicy() throws IOException {
fsView.getStoragePolicy(new Path("/internalDir"));

5
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
View File

@ -511,6 +511,11 @@ public class Hdfs extends AbstractFileSystem {
dfs.checkAccess(getUriPath(path), mode);
}
@Override
public void satisfyStoragePolicy(Path path) throws IOException {
dfs.satisfyStoragePolicy(getUriPath(path));
}
@Override
public void setStoragePolicy(Path path, String policyName) throws IOException {
dfs.setStoragePolicy(getUriPath(path), policyName);

6
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -2914,11 +2914,7 @@ public class DistributedFileSystem extends FileSystem
}
/**
* Set the source path to satisfy storage policy. This API is non-recursive
* in nature, i.e., if the source path is a directory then all the files
* immediately under the directory would be considered for satisfying the
* policy and the sub-directories if any under this path will be skipped.
*
* Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
*/