diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java index 97c5f6a601d..e1d4844b9a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java @@ -140,7 +140,7 @@ public class ErasureCoding { rpcServer.checkOperation(OperationCategory.READ); final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod remoteMethod = new RemoteMethod("getErasureCodingPolicy", new Class[] {String.class}, new RemoteParam()); ErasureCodingPolicy ret = rpcClient.invokeSequential( @@ -153,7 +153,7 @@ public class ErasureCoding { rpcServer.checkOperation(OperationCategory.WRITE); final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod remoteMethod = new RemoteMethod("setErasureCodingPolicy", new Class[] {String.class, String.class}, new RemoteParam(), ecPolicyName); @@ -168,7 +168,7 @@ public class ErasureCoding { rpcServer.checkOperation(OperationCategory.WRITE); final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod remoteMethod = new RemoteMethod("unsetErasureCodingPolicy", new Class[] {String.class}, new RemoteParam()); if (rpcServer.isInvokeConcurrent(src)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java index 4e273c2d416..0a14b99f51e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java @@ -207,7 +207,8 @@ public class RouterClientProtocol implements ClientProtocol { final long length) throws IOException { rpcServer.checkOperation(NameNode.OperationCategory.READ); - List locations = rpcServer.getLocationsForPath(src, false); + List locations = + rpcServer.getLocationsForPath(src, false, false); RemoteMethod remoteMethod = new RemoteMethod("getBlockLocations", new Class[] {String.class, long.class, long.class}, new RemoteParam(), offset, length); @@ -351,7 +352,7 @@ public class RouterClientProtocol implements ClientProtocol { rpcServer.checkOperation(NameNode.OperationCategory.WRITE); final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("recoverLease", new Class[] {String.class, String.class}, new RemoteParam(), clientName); @@ -395,7 +396,7 @@ public class RouterClientProtocol implements ClientProtocol { rpcServer.checkOperation(NameNode.OperationCategory.WRITE); final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("setPermission", new Class[] {String.class, FsPermission.class}, new RemoteParam(), permissions); @@ -412,7 +413,7 @@ public class RouterClientProtocol implements ClientProtocol { rpcServer.checkOperation(NameNode.OperationCategory.WRITE); final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("setOwner", new Class[] {String.class, String.class, String.class}, new RemoteParam(), username, groupname); @@ -531,7 +532,7 @@ public class RouterClientProtocol implements ClientProtocol { rpcServer.checkOperation(NameNode.OperationCategory.READ); final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("getPreferredBlockSize", new Class[] {String.class}, new RemoteParam()); return rpcClient.invokeSequential(locations, method, Long.class, null); @@ -724,7 +725,7 @@ public class RouterClientProtocol implements ClientProtocol { // Locate the dir and fetch the listing final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("getListing", new Class[] {String.class, startAfter.getClass(), boolean.class}, new RemoteParam(), startAfter, needLocation); @@ -821,7 +822,7 @@ public class RouterClientProtocol implements ClientProtocol { rpcServer.checkOperation(NameNode.OperationCategory.READ); final List locations = - rpcServer.getLocationsForPath(src, false); + rpcServer.getLocationsForPath(src, false, false); RemoteMethod method = new RemoteMethod("getFileInfo", new Class[] {String.class}, new RemoteParam()); @@ -859,7 +860,7 @@ public class RouterClientProtocol implements ClientProtocol { rpcServer.checkOperation(NameNode.OperationCategory.READ); final List locations = - rpcServer.getLocationsForPath(src, false); + rpcServer.getLocationsForPath(src, false, false); RemoteMethod method = new RemoteMethod("isFileClosed", new Class[] {String.class}, new RemoteParam()); return rpcClient.invokeSequential(locations, method, Boolean.class, @@ -871,7 +872,7 @@ public class RouterClientProtocol implements ClientProtocol { rpcServer.checkOperation(NameNode.OperationCategory.READ); final List locations = - rpcServer.getLocationsForPath(src, false); + rpcServer.getLocationsForPath(src, false, false); RemoteMethod method = new RemoteMethod("getFileLinkInfo", new Class[] {String.class}, new RemoteParam()); return rpcClient.invokeSequential(locations, method, HdfsFileStatus.class, @@ -883,7 +884,7 @@ public class RouterClientProtocol implements ClientProtocol { boolean needBlockToken) throws IOException { rpcServer.checkOperation(NameNode.OperationCategory.READ); final List locations = - rpcServer.getLocationsForPath(src, false); + rpcServer.getLocationsForPath(src, false, false); RemoteMethod method = new RemoteMethod("getLocatedFileInfo", new Class[] {String.class, boolean.class}, new RemoteParam(), needBlockToken); @@ -1092,7 +1093,7 @@ public class RouterClientProtocol implements ClientProtocol { rpcServer.checkOperation(NameNode.OperationCategory.READ); final List locations = - rpcServer.getLocationsForPath(path, false); + rpcServer.getLocationsForPath(path, false, false); RemoteMethod method = new RemoteMethod("listCorruptFileBlocks", new Class[] {String.class, String.class}, new RemoteParam(), cookie); @@ -1117,7 +1118,7 @@ public class RouterClientProtocol implements ClientProtocol { // Get the summaries from regular files final Collection summaries = new ArrayList<>(); final List locations = - rpcServer.getLocationsForPath(path, false); + rpcServer.getLocationsForPath(path, false, false); final RemoteMethod method = new RemoteMethod("getContentSummary", new Class[] {String.class}, new RemoteParam()); final List> results = @@ -1169,7 +1170,7 @@ public class RouterClientProtocol implements ClientProtocol { rpcServer.checkOperation(NameNode.OperationCategory.WRITE); final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("fsync", new Class[] {String.class, long.class, String.class, long.class }, new RemoteParam(), fileId, clientName, lastBlockLength); @@ -1181,7 +1182,7 @@ public class RouterClientProtocol implements ClientProtocol { rpcServer.checkOperation(NameNode.OperationCategory.WRITE); final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("setTimes", new Class[] {String.class, long.class, long.class}, new RemoteParam(), mtime, atime); @@ -1211,7 +1212,7 @@ public class RouterClientProtocol implements ClientProtocol { rpcServer.checkOperation(NameNode.OperationCategory.READ); final List locations = - rpcServer.getLocationsForPath(path, true); + rpcServer.getLocationsForPath(path, true, false); RemoteMethod method = new RemoteMethod("getLinkTarget", new Class[] {String.class}, new RemoteParam()); return rpcClient.invokeSequential(locations, method, String.class, null); @@ -1309,7 +1310,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("modifyAclEntries", new Class[] {String.class, List.class}, new RemoteParam(), aclSpec); @@ -1327,7 +1328,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("removeAclEntries", new Class[] {String.class, List.class}, new RemoteParam(), aclSpec); @@ -1344,7 +1345,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("removeDefaultAcl", new Class[] {String.class}, new RemoteParam()); if (rpcServer.isInvokeConcurrent(src)) { @@ -1360,7 +1361,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("removeAcl", new Class[] {String.class}, new RemoteParam()); if (rpcServer.isInvokeConcurrent(src)) { @@ -1376,7 +1377,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod( "setAcl", new Class[] {String.class, List.class}, new RemoteParam(), aclSpec); @@ -1393,7 +1394,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(src, false); + rpcServer.getLocationsForPath(src, false, false); RemoteMethod method = new RemoteMethod("getAclStatus", new Class[] {String.class}, new RemoteParam()); return rpcClient.invokeSequential(locations, method, AclStatus.class, null); @@ -1406,7 +1407,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("createEncryptionZone", new Class[] {String.class, String.class}, new RemoteParam(), keyName); @@ -1419,7 +1420,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(src, false); + rpcServer.getLocationsForPath(src, false, false); RemoteMethod method = new RemoteMethod("getEZForPath", new Class[] {String.class}, new RemoteParam()); return rpcClient.invokeSequential( @@ -1453,7 +1454,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("setXAttr", new Class[] {String.class, XAttr.class, EnumSet.class}, new RemoteParam(), xAttr, flag); @@ -1472,7 +1473,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(src, false); + rpcServer.getLocationsForPath(src, false, false); RemoteMethod method = new RemoteMethod("getXAttrs", new Class[] {String.class, List.class}, new RemoteParam(), xAttrs); return (List) rpcClient.invokeSequential( @@ -1486,7 +1487,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(src, false); + rpcServer.getLocationsForPath(src, false, false); RemoteMethod method = new RemoteMethod("listXAttrs", new Class[] {String.class}, new RemoteParam()); return (List) rpcClient.invokeSequential( @@ -1499,7 +1500,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(src, true); + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("removeXAttr", new Class[] {String.class, XAttr.class}, new RemoteParam(), xAttr); if (rpcServer.isInvokeConcurrent(src)) { @@ -1515,7 +1516,7 @@ public class RouterClientProtocol implements ClientProtocol { // TODO handle virtual directories final List locations = - rpcServer.getLocationsForPath(path, true); + rpcServer.getLocationsForPath(path, true, false); RemoteMethod method = new RemoteMethod("checkAccess", new Class[] {String.class, FsAction.class}, new RemoteParam(), mode); @@ -1735,7 +1736,7 @@ public class RouterClientProtocol implements ClientProtocol { throws IOException { final List dstLocations = - rpcServer.getLocationsForPath(dst, true); + rpcServer.getLocationsForPath(dst, true, false); final Map dstMap = new HashMap<>(); Iterator iterator = srcLocations.iterator(); @@ -2026,7 +2027,7 @@ public class RouterClientProtocol implements ClientProtocol { try { if (rpcServer.isPathAll(src)) { List locations; - locations = rpcServer.getLocationsForPath(src, false); + locations = rpcServer.getLocationsForPath(src, false, false); RemoteMethod method = new RemoteMethod("getFileInfo", new Class[] {String.class}, new RemoteParam()); HdfsFileStatus fileStatus = rpcClient.invokeSequential(locations, diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStoragePolicy.java index a4538b0e6bd..33203dcdb99 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStoragePolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStoragePolicy.java @@ -45,7 +45,8 @@ public class RouterStoragePolicy { throws IOException { rpcServer.checkOperation(NameNode.OperationCategory.WRITE); - List locations = rpcServer.getLocationsForPath(src, true); + List locations = + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("setStoragePolicy", new Class[] {String.class, String.class}, new RemoteParam(), @@ -67,7 +68,8 @@ public class RouterStoragePolicy { public void unsetStoragePolicy(String src) throws IOException { rpcServer.checkOperation(NameNode.OperationCategory.WRITE, true); - List locations = rpcServer.getLocationsForPath(src, true); + List locations = + rpcServer.getLocationsForPath(src, true, false); RemoteMethod method = new RemoteMethod("unsetStoragePolicy", new Class[] {String.class}, new RemoteParam()); @@ -82,7 +84,8 @@ public class RouterStoragePolicy { throws IOException { rpcServer.checkOperation(NameNode.OperationCategory.READ, true); - List locations = rpcServer.getLocationsForPath(path, false); + List locations = + rpcServer.getLocationsForPath(path, false, false); RemoteMethod method = new RemoteMethod("getStoragePolicy", new Class[] {String.class}, new RemoteParam()); @@ -92,7 +95,8 @@ public class RouterStoragePolicy { public void satisfyStoragePolicy(String path) throws IOException { rpcServer.checkOperation(NameNode.OperationCategory.READ, true); - List locations = rpcServer.getLocationsForPath(path, true); + List locations = + rpcServer.getLocationsForPath(path, true, false); RemoteMethod method = new RemoteMethod("satisfyStoragePolicy", new Class[] {String.class}, new RemoteParam()); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java index abcbe8fdbdc..81d5023cfc0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; @@ -34,7 +35,9 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.QuotaUsage; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.ClientProtocol; @@ -792,4 +795,34 @@ public class TestRouterQuota { assertEquals(nsQuota, quota1.getQuota()); assertEquals(ssQuota, quota1.getSpaceQuota()); } + + @Test + public void testNoQuotaaExceptionForUnrelatedOperations() throws Exception { + FileSystem nnFs = nnContext1.getFileSystem(); + DistributedFileSystem routerFs = + (DistributedFileSystem) routerContext.getFileSystem(); + Path path = new Path("/quota"); + nnFs.mkdirs(new Path("/dir")); + MountTable mountTable1 = MountTable.newInstance("/quota", + Collections.singletonMap("ns0", "/dir")); + mountTable1.setQuota(new RouterQuotaUsage.Builder().quota(0).build()); + addMountTable(mountTable1); + routerFs.mkdirs(new Path("/quota/1")); + routerContext.getRouter().getQuotaCacheUpdateService().periodicInvoke(); + + // Quota check for related operation. + intercept(NSQuotaExceededException.class, + "The NameSpace quota (directories and files) is exceeded", + () -> routerFs.mkdirs(new Path("/quota/2"))); + + //Quotas shouldn't be checked for unrelated operations. + routerFs.setStoragePolicy(path, "COLD"); + routerFs.setErasureCodingPolicy(path, "RS-6-3-1024k"); + routerFs.unsetErasureCodingPolicy(path); + routerFs.setPermission(path, new FsPermission((short) 01777)); + routerFs.setOwner(path, "user", "group"); + routerFs.setTimes(path, 1L, 1L); + routerFs.listStatus(path); + routerFs.getContentSummary(path); + } }