HADOOP-13272. ViewFileSystem should support storage policy related API. Contributed by Peter Shi
This commit is contained in:
parent
5fda66e4a3
commit
22a9a6b0d4
|
@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
|||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.BlockLocation;
|
||||
import org.apache.hadoop.fs.BlockStoragePolicySpi;
|
||||
import org.apache.hadoop.fs.ContentSummary;
|
||||
import org.apache.hadoop.fs.CreateFlag;
|
||||
import org.apache.hadoop.fs.FSDataInputStream;
|
||||
|
@ -39,12 +40,12 @@ import org.apache.hadoop.fs.FsServerDefaults;
|
|||
import org.apache.hadoop.fs.FsStatus;
|
||||
import org.apache.hadoop.fs.LocatedFileStatus;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.XAttrSetFlag;
|
||||
import org.apache.hadoop.fs.QuotaUsage;
|
||||
import org.apache.hadoop.fs.RemoteIterator;
|
||||
import org.apache.hadoop.fs.XAttrSetFlag;
|
||||
import org.apache.hadoop.fs.permission.AclEntry;
|
||||
import org.apache.hadoop.fs.permission.AclStatus;
|
||||
import org.apache.hadoop.fs.permission.FsAction;
|
||||
import org.apache.hadoop.fs.QuotaUsage;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.security.AccessControlException;
|
||||
import org.apache.hadoop.util.Progressable;
|
||||
|
@ -425,5 +426,21 @@ class ChRootedFileSystem extends FilterFileSystem {
|
|||
@Override
|
||||
public FsServerDefaults getServerDefaults(Path f) throws IOException {
|
||||
return super.getServerDefaults(fullPath(f));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
|
||||
return super.getStoragePolicy(fullPath(src));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setStoragePolicy(Path src, String policyName) throws IOException {
|
||||
super.setStoragePolicy(fullPath(src), policyName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void unsetStoragePolicy(Path src) throws IOException {
|
||||
super.unsetStoragePolicy(fullPath(src));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -24,6 +24,7 @@ import java.io.IOException;
|
|||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
@ -35,6 +36,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
|||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.BlockLocation;
|
||||
import org.apache.hadoop.fs.BlockStoragePolicySpi;
|
||||
import org.apache.hadoop.fs.ContentSummary;
|
||||
import org.apache.hadoop.fs.CreateFlag;
|
||||
import org.apache.hadoop.fs.FSDataInputStream;
|
||||
|
@ -48,6 +50,7 @@ import org.apache.hadoop.fs.FsServerDefaults;
|
|||
import org.apache.hadoop.fs.LocatedFileStatus;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.PathFilter;
|
||||
import org.apache.hadoop.fs.QuotaUsage;
|
||||
import org.apache.hadoop.fs.RemoteIterator;
|
||||
import org.apache.hadoop.fs.UnsupportedFileSystemException;
|
||||
import org.apache.hadoop.fs.XAttrSetFlag;
|
||||
|
@ -56,7 +59,6 @@ import org.apache.hadoop.fs.permission.AclStatus;
|
|||
import org.apache.hadoop.fs.permission.AclUtil;
|
||||
import org.apache.hadoop.fs.permission.FsAction;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.fs.QuotaUsage;
|
||||
import org.apache.hadoop.fs.viewfs.InodeTree.INode;
|
||||
import org.apache.hadoop.fs.viewfs.InodeTree.INodeLink;
|
||||
import org.apache.hadoop.security.AccessControlException;
|
||||
|
@ -760,6 +762,43 @@ public class ViewFileSystem extends FileSystem {
|
|||
res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setStoragePolicy(Path src, String policyName) throws IOException {
|
||||
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
|
||||
true);
|
||||
res.targetFileSystem.setStoragePolicy(res.remainingPath, policyName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void unsetStoragePolicy(Path src) throws IOException {
|
||||
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
|
||||
true);
|
||||
res.targetFileSystem.unsetStoragePolicy(res.remainingPath);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
|
||||
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
|
||||
true);
|
||||
return res.targetFileSystem.getStoragePolicy(res.remainingPath);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
|
||||
throws IOException {
|
||||
Collection<BlockStoragePolicySpi> allPolicies = new HashSet<>();
|
||||
for (FileSystem fs : getChildFileSystems()) {
|
||||
try {
|
||||
Collection<? extends BlockStoragePolicySpi> policies =
|
||||
fs.getAllStoragePolicies();
|
||||
allPolicies.addAll(policies);
|
||||
} catch (UnsupportedOperationException e) {
|
||||
// ignored
|
||||
}
|
||||
}
|
||||
return allPolicies;
|
||||
}
|
||||
|
||||
/*
|
||||
* An instance of this class represents an internal dir of the viewFs
|
||||
* that is internal dir of the mount table.
|
||||
|
@ -1079,5 +1118,39 @@ public class ViewFileSystem extends FileSystem {
|
|||
public QuotaUsage getQuotaUsage(Path f) throws IOException {
|
||||
throw new NotInMountpointException(f, "getQuotaUsage");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setStoragePolicy(Path src, String policyName)
|
||||
throws IOException {
|
||||
checkPathIsSlash(src);
|
||||
throw readOnlyMountTable("setStoragePolicy", src);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void unsetStoragePolicy(Path src) throws IOException {
|
||||
checkPathIsSlash(src);
|
||||
throw readOnlyMountTable("unsetStoragePolicy", src);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
|
||||
throw new NotInMountpointException(src, "getStoragePolicy");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
|
||||
throws IOException {
|
||||
Collection<BlockStoragePolicySpi> allPolicies = new HashSet<>();
|
||||
for (FileSystem fs : getChildFileSystems()) {
|
||||
try {
|
||||
Collection<? extends BlockStoragePolicySpi> policies =
|
||||
fs.getAllStoragePolicies();
|
||||
allPolicies.addAll(policies);
|
||||
} catch (UnsupportedOperationException e) {
|
||||
// ignored
|
||||
}
|
||||
}
|
||||
return allPolicies;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -468,4 +468,69 @@ public class TestChRootedFileSystem {
|
|||
verify(mockFs).renameSnapshot(chRootedSnapRootPath, "snapOldName",
|
||||
"snapNewName");
|
||||
}
|
||||
|
||||
@Test(timeout = 30000)
|
||||
public void testSetStoragePolicy() throws Exception {
|
||||
Path storagePolicyPath = new Path("/storagePolicy");
|
||||
Path chRootedStoragePolicyPath = new Path("/a/b/storagePolicy");
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
|
||||
|
||||
URI chrootUri = URI.create("mockfs://foo/a/b");
|
||||
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
|
||||
FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
|
||||
.getRawFileSystem();
|
||||
|
||||
chrootFs.setStoragePolicy(storagePolicyPath, "HOT");
|
||||
verify(mockFs).setStoragePolicy(chRootedStoragePolicyPath, "HOT");
|
||||
}
|
||||
|
||||
@Test(timeout = 30000)
|
||||
public void testUnsetStoragePolicy() throws Exception {
|
||||
Path storagePolicyPath = new Path("/storagePolicy");
|
||||
Path chRootedStoragePolicyPath = new Path("/a/b/storagePolicy");
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
|
||||
|
||||
URI chrootUri = URI.create("mockfs://foo/a/b");
|
||||
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
|
||||
FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
|
||||
.getRawFileSystem();
|
||||
|
||||
chrootFs.unsetStoragePolicy(storagePolicyPath);
|
||||
verify(mockFs).unsetStoragePolicy(chRootedStoragePolicyPath);
|
||||
}
|
||||
|
||||
@Test(timeout = 30000)
|
||||
public void testGetStoragePolicy() throws Exception {
|
||||
Path storagePolicyPath = new Path("/storagePolicy");
|
||||
Path chRootedStoragePolicyPath = new Path("/a/b/storagePolicy");
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
|
||||
|
||||
URI chrootUri = URI.create("mockfs://foo/a/b");
|
||||
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
|
||||
FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
|
||||
.getRawFileSystem();
|
||||
|
||||
chrootFs.getStoragePolicy(storagePolicyPath);
|
||||
verify(mockFs).getStoragePolicy(chRootedStoragePolicyPath);
|
||||
}
|
||||
|
||||
@Test(timeout = 30000)
|
||||
public void testGetAllStoragePolicy() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
|
||||
|
||||
URI chrootUri = URI.create("mockfs://foo/a/b");
|
||||
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
|
||||
FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
|
||||
.getRawFileSystem();
|
||||
|
||||
chrootFs.getAllStoragePolicies();
|
||||
verify(mockFs).getAllStoragePolicies();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,42 +19,41 @@ package org.apache.hadoop.fs.viewfs;
|
|||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.BlockLocation;
|
||||
import org.apache.hadoop.fs.BlockStoragePolicySpi;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.FileSystemTestHelper;
|
||||
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
|
||||
import org.apache.hadoop.fs.permission.AclEntry;
|
||||
import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FsConstants;
|
||||
import org.apache.hadoop.fs.LocatedFileStatus;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.RemoteIterator;
|
||||
import org.apache.hadoop.fs.permission.AclEntry;
|
||||
import org.apache.hadoop.fs.permission.AclStatus;
|
||||
import org.apache.hadoop.fs.permission.AclUtil;
|
||||
import org.apache.hadoop.fs.RemoteIterator;
|
||||
import org.apache.hadoop.fs.permission.FsAction;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.fs.viewfs.ConfigUtil;
|
||||
import org.apache.hadoop.fs.viewfs.ViewFileSystem;
|
||||
import org.apache.hadoop.fs.viewfs.ViewFileSystem.MountPoint;
|
||||
import org.apache.hadoop.fs.viewfs.ViewFileSystem;
|
||||
import org.apache.hadoop.security.AccessControlException;
|
||||
import org.apache.hadoop.security.Credentials;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.token.Token;
|
||||
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
|
||||
import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
|
||||
/**
|
||||
|
@ -885,4 +884,35 @@ abstract public class ViewFileSystemBaseTest {
|
|||
public void testInternalDeleteSnapshot() throws IOException {
|
||||
fsView.deleteSnapshot(new Path("/internalDir"), "snap1");
|
||||
}
|
||||
|
||||
@Test(expected = AccessControlException.class)
|
||||
public void testInternalSetStoragePolicy() throws IOException {
|
||||
fsView.setStoragePolicy(new Path("/internalDir"), "HOT");
|
||||
}
|
||||
|
||||
@Test(expected = AccessControlException.class)
|
||||
public void testInternalUnsetStoragePolicy() throws IOException {
|
||||
fsView.unsetStoragePolicy(new Path("/internalDir"));
|
||||
}
|
||||
|
||||
@Test(expected = NotInMountpointException.class)
|
||||
public void testInternalgetStoragePolicy() throws IOException {
|
||||
fsView.getStoragePolicy(new Path("/internalDir"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInternalGetAllStoragePolicies() throws IOException {
|
||||
Collection<? extends BlockStoragePolicySpi> policies =
|
||||
fsView.getAllStoragePolicies();
|
||||
for (FileSystem fs : fsView.getChildFileSystems()) {
|
||||
try {
|
||||
for (BlockStoragePolicySpi s : fs.getAllStoragePolicies()) {
|
||||
assertTrue("Missing policy: " + s, policies.contains(s));
|
||||
}
|
||||
} catch (UnsupportedOperationException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue