Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14490. RBF: Remove unnecessary quota checks. Contributed by Ayush Saxena.

Browse Source
This commit is contained in:
Giovanni Matteo Fumarola 2019-05-15 15:22:36 -07:00 committed by Brahma Reddy Battula
parent 4afe588ba2
commit a53d678ad8
4 changed files with 75 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java
View File

@ -140,7 +140,7 @@ public ErasureCodingPolicy getErasureCodingPolicy(String src)
rpcServer.checkOperation(OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod remoteMethod = new RemoteMethod("getErasureCodingPolicy",
new Class<?>[] {String.class}, new RemoteParam());
ErasureCodingPolicy ret = rpcClient.invokeSequential(
@ -153,7 +153,7 @@ public void setErasureCodingPolicy(String src, String ecPolicyName)
rpcServer.checkOperation(OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod remoteMethod = new RemoteMethod("setErasureCodingPolicy",
new Class<?>[] {String.class, String.class},
new RemoteParam(), ecPolicyName);
@ -168,7 +168,7 @@ public void unsetErasureCodingPolicy(String src) throws IOException {
rpcServer.checkOperation(OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod remoteMethod = new RemoteMethod("unsetErasureCodingPolicy",
new Class<?>[] {String.class}, new RemoteParam());
if (rpcServer.isInvokeConcurrent(src)) {

61
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
View File

@ -207,7 +207,8 @@ public LocatedBlocks getBlockLocations(String src, final long offset,
final long length) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, false);
List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod remoteMethod = new RemoteMethod("getBlockLocations",
new Class<?>[] {String.class, long.class, long.class},
new RemoteParam(), offset, length);
@ -351,7 +352,7 @@ public boolean recoverLease(String src, String clientName)
rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("recoverLease",
new Class<?>[] {String.class, String.class}, new RemoteParam(),
clientName);
@ -395,7 +396,7 @@ public void setPermission(String src, FsPermission permissions)
rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("setPermission",
new Class<?>[] {String.class, FsPermission.class},
new RemoteParam(), permissions);
@ -412,7 +413,7 @@ public void setOwner(String src, String username, String groupname)
rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("setOwner",
new Class<?>[] {String.class, String.class, String.class},
new RemoteParam(), username, groupname);
@ -531,7 +532,7 @@ public long getPreferredBlockSize(String src) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("getPreferredBlockSize",
new Class<?>[] {String.class}, new RemoteParam());
return rpcClient.invokeSequential(locations, method, Long.class, null);
@ -724,7 +725,7 @@ public DirectoryListing getListing(String src, byte[] startAfter,
// Locate the dir and fetch the listing
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("getListing",
new Class<?>[] {String.class, startAfter.getClass(), boolean.class},
new RemoteParam(), startAfter, needLocation);
@ -821,7 +822,7 @@ public HdfsFileStatus getFileInfo(String src) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getFileInfo",
new Class<?>[] {String.class}, new RemoteParam());
@ -859,7 +860,7 @@ public boolean isFileClosed(String src) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("isFileClosed",
new Class<?>[] {String.class}, new RemoteParam());
return rpcClient.invokeSequential(locations, method, Boolean.class,
@ -871,7 +872,7 @@ public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getFileLinkInfo",
new Class<?>[] {String.class}, new RemoteParam());
return rpcClient.invokeSequential(locations, method, HdfsFileStatus.class,
@ -883,7 +884,7 @@ public HdfsLocatedFileStatus getLocatedFileInfo(String src,
boolean needBlockToken) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getLocatedFileInfo",
new Class<?>[] {String.class, boolean.class}, new RemoteParam(),
needBlockToken);
@ -1092,7 +1093,7 @@ public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie)
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(path, false);
rpcServer.getLocationsForPath(path, false, false);
RemoteMethod method = new RemoteMethod("listCorruptFileBlocks",
new Class<?>[] {String.class, String.class},
new RemoteParam(), cookie);
@ -1117,7 +1118,7 @@ public ContentSummary getContentSummary(String path) throws IOException {
// Get the summaries from regular files
final Collection<ContentSummary> summaries = new ArrayList<>();
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(path, false);
rpcServer.getLocationsForPath(path, false, false);
final RemoteMethod method = new RemoteMethod("getContentSummary",
new Class<?>[] {String.class}, new RemoteParam());
final List<RemoteResult<RemoteLocation, ContentSummary>> results =
@ -1169,7 +1170,7 @@ public void fsync(String src, long fileId, String clientName,
rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("fsync",
new Class<?>[] {String.class, long.class, String.class, long.class },
new RemoteParam(), fileId, clientName, lastBlockLength);
@ -1181,7 +1182,7 @@ public void setTimes(String src, long mtime, long atime) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("setTimes",
new Class<?>[] {String.class, long.class, long.class},
new RemoteParam(), mtime, atime);
@ -1211,7 +1212,7 @@ public String getLinkTarget(String path) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(path, true);
rpcServer.getLocationsForPath(path, true, false);
RemoteMethod method = new RemoteMethod("getLinkTarget",
new Class<?>[] {String.class}, new RemoteParam());
return rpcClient.invokeSequential(locations, method, String.class, null);
@ -1309,7 +1310,7 @@ public void modifyAclEntries(String src, List<AclEntry> aclSpec)
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("modifyAclEntries",
new Class<?>[] {String.class, List.class},
new RemoteParam(), aclSpec);
@ -1327,7 +1328,7 @@ public void removeAclEntries(String src, List<AclEntry> aclSpec)
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("removeAclEntries",
new Class<?>[] {String.class, List.class},
new RemoteParam(), aclSpec);
@ -1344,7 +1345,7 @@ public void removeDefaultAcl(String src) throws IOException {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("removeDefaultAcl",
new Class<?>[] {String.class}, new RemoteParam());
if (rpcServer.isInvokeConcurrent(src)) {
@ -1360,7 +1361,7 @@ public void removeAcl(String src) throws IOException {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("removeAcl",
new Class<?>[] {String.class}, new RemoteParam());
if (rpcServer.isInvokeConcurrent(src)) {
@ -1376,7 +1377,7 @@ public void setAcl(String src, List<AclEntry> aclSpec) throws IOException {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod(
"setAcl", new Class<?>[] {String.class, List.class},
new RemoteParam(), aclSpec);
@ -1393,7 +1394,7 @@ public AclStatus getAclStatus(String src) throws IOException {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getAclStatus",
new Class<?>[] {String.class}, new RemoteParam());
return rpcClient.invokeSequential(locations, method, AclStatus.class, null);
@ -1406,7 +1407,7 @@ public void createEncryptionZone(String src, String keyName)
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("createEncryptionZone",
new Class<?>[] {String.class, String.class},
new RemoteParam(), keyName);
@ -1419,7 +1420,7 @@ public EncryptionZone getEZForPath(String src) throws IOException {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getEZForPath",
new Class<?>[] {String.class}, new RemoteParam());
return rpcClient.invokeSequential(
@ -1453,7 +1454,7 @@ public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("setXAttr",
new Class<?>[] {String.class, XAttr.class, EnumSet.class},
new RemoteParam(), xAttr, flag);
@ -1472,7 +1473,7 @@ public List<XAttr> getXAttrs(String src, List<XAttr> xAttrs)
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getXAttrs",
new Class<?>[] {String.class, List.class}, new RemoteParam(), xAttrs);
return (List<XAttr>) rpcClient.invokeSequential(
@ -1486,7 +1487,7 @@ public List<XAttr> listXAttrs(String src) throws IOException {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("listXAttrs",
new Class<?>[] {String.class}, new RemoteParam());
return (List<XAttr>) rpcClient.invokeSequential(
@ -1499,7 +1500,7 @@ public void removeXAttr(String src, XAttr xAttr) throws IOException {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("removeXAttr",
new Class<?>[] {String.class, XAttr.class}, new RemoteParam(), xAttr);
if (rpcServer.isInvokeConcurrent(src)) {
@ -1515,7 +1516,7 @@ public void checkAccess(String path, FsAction mode) throws IOException {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(path, true);
rpcServer.getLocationsForPath(path, true, false);
RemoteMethod method = new RemoteMethod("checkAccess",
new Class<?>[] {String.class, FsAction.class},
new RemoteParam(), mode);
@ -1735,7 +1736,7 @@ private RemoteParam getRenameDestinations(
throws IOException {
final List<RemoteLocation> dstLocations =
rpcServer.getLocationsForPath(dst, true);
rpcServer.getLocationsForPath(dst, true, false);
final Map<RemoteLocation, String> dstMap = new HashMap<>();
Iterator<RemoteLocation> iterator = srcLocations.iterator();
@ -2026,7 +2027,7 @@ boolean isMultiDestDirectory(String src) throws IOException {
try {
if (rpcServer.isPathAll(src)) {
List<RemoteLocation> locations;
locations = rpcServer.getLocationsForPath(src, false);
locations = rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getFileInfo",
new Class<?>[] {String.class}, new RemoteParam());
HdfsFileStatus fileStatus = rpcClient.invokeSequential(locations,

12
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStoragePolicy.java
View File

@ -45,7 +45,8 @@ public void setStoragePolicy(String src, String policyName)
throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, true);
List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("setStoragePolicy",
new Class<?>[] {String.class, String.class},
new RemoteParam(),
@ -67,7 +68,8 @@ public BlockStoragePolicy[] getStoragePolicies() throws IOException {
public void unsetStoragePolicy(String src) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.WRITE, true);
List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, true);
List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("unsetStoragePolicy",
new Class<?>[] {String.class},
new RemoteParam());
@ -82,7 +84,8 @@ public BlockStoragePolicy getStoragePolicy(String path)
throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ, true);
List<RemoteLocation> locations = rpcServer.getLocationsForPath(path, false);
List<RemoteLocation> locations =
rpcServer.getLocationsForPath(path, false, false);
RemoteMethod method = new RemoteMethod("getStoragePolicy",
new Class<?>[] {String.class},
new RemoteParam());
@ -92,7 +95,8 @@ public BlockStoragePolicy getStoragePolicy(String path)
public void satisfyStoragePolicy(String path) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ, true);
List<RemoteLocation> locations = rpcServer.getLocationsForPath(path, true);
List<RemoteLocation> locations =
rpcServer.getLocationsForPath(path, true, false);
RemoteMethod method = new RemoteMethod("satisfyStoragePolicy",
new Class<?>[] {String.class},
new RemoteParam());

33
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNull;
@ -34,7 +35,9 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
@ -792,4 +795,34 @@ public void testSetQuotaNotMountTable() throws Exception {
assertEquals(nsQuota, quota1.getQuota());
assertEquals(ssQuota, quota1.getSpaceQuota());
}
@Test
public void testNoQuotaaExceptionForUnrelatedOperations() throws Exception {
FileSystem nnFs = nnContext1.getFileSystem();
DistributedFileSystem routerFs =
(DistributedFileSystem) routerContext.getFileSystem();
Path path = new Path("/quota");
nnFs.mkdirs(new Path("/dir"));
MountTable mountTable1 = MountTable.newInstance("/quota",
Collections.singletonMap("ns0", "/dir"));
mountTable1.setQuota(new RouterQuotaUsage.Builder().quota(0).build());
addMountTable(mountTable1);
routerFs.mkdirs(new Path("/quota/1"));
routerContext.getRouter().getQuotaCacheUpdateService().periodicInvoke();
// Quota check for related operation.
intercept(NSQuotaExceededException.class,
"The NameSpace quota (directories and files) is exceeded",
() -> routerFs.mkdirs(new Path("/quota/2")));
//Quotas shouldn't be checked for unrelated operations.
routerFs.setStoragePolicy(path, "COLD");
routerFs.setErasureCodingPolicy(path, "RS-6-3-1024k");
routerFs.unsetErasureCodingPolicy(path);
routerFs.setPermission(path, new FsPermission((short) 01777));
routerFs.setOwner(path, "user", "group");
routerFs.setTimes(path, 1L, 1L);
routerFs.listStatus(path);
routerFs.getContentSummary(path);
}
}