From 601b5038954bd3b44c02e58a2fbaa15082d8b54d Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 23 Jul 2015 10:13:04 -0700 Subject: [PATCH] HADOOP-12161. Add getStoragePolicy API to the FileSystem interface. (Contributed by Brahma Reddy Battula) --- .../hadoop-common/CHANGES.txt | 3 +++ .../apache/hadoop/fs/AbstractFileSystem.java | 13 ++++++++++++ .../org/apache/hadoop/fs/FileContext.java | 20 +++++++++++++++++++ .../java/org/apache/hadoop/fs/FileSystem.java | 13 ++++++++++++ .../apache/hadoop/fs/FilterFileSystem.java | 6 ++++++ .../java/org/apache/hadoop/fs/FilterFs.java | 6 ++++++ .../apache/hadoop/fs/viewfs/ChRootedFs.java | 6 ++++++ .../org/apache/hadoop/fs/viewfs/ViewFs.java | 15 ++++++++++++++ .../apache/hadoop/fs/TestHarFileSystem.java | 3 +++ .../main/java/org/apache/hadoop/fs/Hdfs.java | 5 +++++ .../org/apache/hadoop/hdfs/DFSClient.java | 18 +++++++++++++++++ .../hadoop/hdfs/DistributedFileSystem.java | 19 ++++++++++++++++++ .../hadoop/hdfs/TestBlockStoragePolicy.java | 17 ++++++++++++++++ 13 files changed, 144 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 6320dc3e056..b7d0445ea0c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -211,6 +211,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12184. Remove unused Linux-specific constants in NativeIO (Martin Walsh via Colin P. McCabe) + HADOOP-12161. Add getStoragePolicy API to the FileSystem interface. + (Brahma Reddy Battula via Arpit Agarwal) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index cb3fb862325..2bc38591509 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -1236,6 +1236,19 @@ public abstract class AbstractFileSystem { + " doesn't support setStoragePolicy"); } + /** + * Retrieve the storage policy for a given file or directory. + * + * @param src file or directory path. + * @return storage policy for give file. + * @throws IOException + */ + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException { + throw new UnsupportedOperationException(getClass().getSimpleName() + + " doesn't support getStoragePolicy"); + } + /** * Retrieve all the storage policies supported by this file system. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index f6fe054c47d..4321ed252c7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -49,6 +49,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT; + import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RpcClientException; import org.apache.hadoop.ipc.RpcServerException; @@ -2701,6 +2702,25 @@ public class FileContext { }.resolve(this, absF); } + /** + * Query the effective storage policy ID for the given file or directory. + * + * @param src file or directory path. + * @return storage policy for give file. + * @throws IOException + */ + public BlockStoragePolicySpi getStoragePolicy(Path path) throws IOException { + final Path absF = fixRelativePart(path); + return new FSLinkResolver() { + @Override + public BlockStoragePolicySpi next(final AbstractFileSystem fs, + final Path p) + throws IOException { + return fs.getStoragePolicy(p); + } + }.resolve(this, absF); + } + /** * Retrieve all the storage policies supported by this file system. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index b94f65c938a..29956f73ecf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -2629,6 +2629,19 @@ public abstract class FileSystem extends Configured implements Closeable { + " doesn't support setStoragePolicy"); } + /** + * Query the effective storage policy ID for the given file or directory. + * + * @param src file or directory path. + * @return storage policy for give file. + * @throws IOException + */ + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException { + throw new UnsupportedOperationException(getClass().getSimpleName() + + " doesn't support getStoragePolicy"); + } + /** * Retrieve all the storage policies supported by this file system. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index 11f3b23c33c..815ef6974d6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -627,6 +627,12 @@ public class FilterFileSystem extends FileSystem { fs.setStoragePolicy(src, policyName); } + @Override + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException { + return fs.getStoragePolicy(src); + } + @Override public Collection getAllStoragePolicies() throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java index 539b26e166a..248377c9727 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java @@ -405,6 +405,12 @@ public abstract class FilterFs extends AbstractFileSystem { myFs.setStoragePolicy(path, policyName); } + @Override + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException { + return myFs.getStoragePolicy(src); + } + @Override public Collection getAllStoragePolicies() throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index 4e5a0d55e1f..568b9a0f861 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -385,6 +385,12 @@ class ChRootedFs extends AbstractFileSystem { myFs.setStoragePolicy(fullPath(path), policyName); } + @Override + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException { + return myFs.getStoragePolicy(src); + } + @Override public Collection getAllStoragePolicies() throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index bec292cd5b9..6f05e77dd01 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -34,6 +34,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.BlockStoragePolicySpi; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -748,6 +749,20 @@ public class ViewFs extends AbstractFileSystem { res.targetFileSystem.setStoragePolicy(res.remainingPath, policyName); } + /** + * Retrieve the storage policy for a given file or directory. + * + * @param src file or directory path. + * @return storage policy for give file. + * @throws IOException + */ + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException { + InodeTree.ResolveResult res = + fsState.resolve(getUriPath(src), true); + return res.targetFileSystem.getStoragePolicy(res.remainingPath); + } + /* * An instance of this class represents an internal dir of the viewFs * ie internal dir of the mount table. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java index 46f24fc6804..1710ba4a4bf 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java @@ -210,6 +210,9 @@ public class TestHarFileSystem { public void setStoragePolicy(Path src, String policyName) throws IOException; + public BlockStoragePolicySpi getStoragePolicy(final Path src) + throws IOException; + public Collection getAllStoragePolicies() throws IOException; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java index 3f78b3182a0..ba5687cb050 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -472,6 +472,11 @@ public class Hdfs extends AbstractFileSystem { dfs.setStoragePolicy(getUriPath(path), policyName); } + @Override + public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException { + return dfs.getStoragePolicy(getUriPath(src)); + } + @Override public Collection getAllStoragePolicies() throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 11a5e9daf19..160c7a082bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -1567,6 +1567,24 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } } + /** + * @return Get the storage policy for specified path + */ + public BlockStoragePolicy getStoragePolicy(String path) throws IOException { + HdfsFileStatus status = getFileInfo(path); + if (status == null) { + throw new FileNotFoundException("File does not exist: " + path); + } + byte storagePolicyId = status.getStoragePolicy(); + BlockStoragePolicy[] policies = getStoragePolicies(); + for (BlockStoragePolicy policy : policies) { + if (policy.getId() == storagePolicyId) { + return policy; + } + } + return null; + } + /** * @return All the existing storage policies */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index a206b3823cb..e37904508c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -555,6 +555,25 @@ public class DistributedFileSystem extends FileSystem { }.resolve(this, absF); } + @Override + public BlockStoragePolicySpi getStoragePolicy(Path path) throws IOException { + statistics.incrementReadOps(1); + Path absF = fixRelativePart(path); + + return new FileSystemLinkResolver() { + @Override + public BlockStoragePolicySpi doCall(final Path p) throws IOException { + return getClient().getStoragePolicy(getPathName(p)); + } + + @Override + public BlockStoragePolicySpi next(final FileSystem fs, final Path p) + throws IOException, UnresolvedLinkException { + return fs.getStoragePolicy(p); + } + }.resolve(this, absF); + } + @Override public Collection getAllStoragePolicies() throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java index 0cf9124a779..d11b06dd1ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java @@ -871,9 +871,25 @@ public class TestBlockStoragePolicy { GenericTestUtils.assertExceptionContains(invalidPath.toString(), e); } + try { + fs.getStoragePolicy(invalidPath); + Assert.fail("Should throw a FileNotFoundException"); + } catch (FileNotFoundException e) { + GenericTestUtils.assertExceptionContains(invalidPath.toString(), e); + } + fs.setStoragePolicy(fooFile, HdfsServerConstants.COLD_STORAGE_POLICY_NAME); fs.setStoragePolicy(barDir, HdfsServerConstants.WARM_STORAGE_POLICY_NAME); fs.setStoragePolicy(barFile2, HdfsServerConstants.HOT_STORAGE_POLICY_NAME); + Assert.assertEquals("File storage policy should be COLD", + HdfsServerConstants.COLD_STORAGE_POLICY_NAME, + fs.getStoragePolicy(fooFile).getName()); + Assert.assertEquals("File storage policy should be WARM", + HdfsServerConstants.WARM_STORAGE_POLICY_NAME, + fs.getStoragePolicy(barDir).getName()); + Assert.assertEquals("File storage policy should be HOT", + HdfsServerConstants.HOT_STORAGE_POLICY_NAME, + fs.getStoragePolicy(barFile2).getName()); dirList = fs.getClient().listPaths(dir.toString(), HdfsFileStatus.EMPTY_NAME).getPartialListing(); @@ -1330,4 +1346,5 @@ public class TestBlockStoragePolicy { Assert.assertEquals(StorageType.ARCHIVE, i.next().getKey()); } } + }