Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14490. RBF: Remove unnecessary quota checks. Contributed by Ayush Saxena.

This commit is contained in:
Giovanni Matteo Fumarola 2019-05-15 15:22:36 -07:00 committed by Brahma Reddy Battula
parent 62fa53a01d
commit 3deb5d345f
4 changed files with 75 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java
View File

@ -140,7 +140,7 @@ public class ErasureCoding {
rpcServer.checkOperation(OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod remoteMethod = new RemoteMethod("getErasureCodingPolicy",
new Class<?>[] {String.class}, new RemoteParam());
ErasureCodingPolicy ret = rpcClient.invokeSequential(
@ -153,7 +153,7 @@ public class ErasureCoding {
rpcServer.checkOperation(OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod remoteMethod = new RemoteMethod("setErasureCodingPolicy",
new Class<?>[] {String.class, String.class},
new RemoteParam(), ecPolicyName);
@ -168,7 +168,7 @@ public class ErasureCoding {
rpcServer.checkOperation(OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod remoteMethod = new RemoteMethod("unsetErasureCodingPolicy",
new Class<?>[] {String.class}, new RemoteParam());
if (rpcServer.isInvokeConcurrent(src)) {

61
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
View File

@ -207,7 +207,8 @@ public class RouterClientProtocol implements ClientProtocol {
final long length) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, false);
List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod remoteMethod = new RemoteMethod("getBlockLocations",
new Class<?>[] {String.class, long.class, long.class},
new RemoteParam(), offset, length);
@ -351,7 +352,7 @@ public class RouterClientProtocol implements ClientProtocol {
rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("recoverLease",
new Class<?>[] {String.class, String.class}, new RemoteParam(),
clientName);
@ -395,7 +396,7 @@ public class RouterClientProtocol implements ClientProtocol {
rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("setPermission",
new Class<?>[] {String.class, FsPermission.class},
new RemoteParam(), permissions);
@ -412,7 +413,7 @@ public class RouterClientProtocol implements ClientProtocol {
rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("setOwner",
new Class<?>[] {String.class, String.class, String.class},
new RemoteParam(), username, groupname);
@ -531,7 +532,7 @@ public class RouterClientProtocol implements ClientProtocol {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("getPreferredBlockSize",
new Class<?>[] {String.class}, new RemoteParam());
return rpcClient.invokeSequential(locations, method, Long.class, null);
@ -724,7 +725,7 @@ public class RouterClientProtocol implements ClientProtocol {
// Locate the dir and fetch the listing
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("getListing",
new Class<?>[] {String.class, startAfter.getClass(), boolean.class},
new RemoteParam(), startAfter, needLocation);
@ -821,7 +822,7 @@ public class RouterClientProtocol implements ClientProtocol {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getFileInfo",
new Class<?>[] {String.class}, new RemoteParam());
@ -859,7 +860,7 @@ public class RouterClientProtocol implements ClientProtocol {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("isFileClosed",
new Class<?>[] {String.class}, new RemoteParam());
return rpcClient.invokeSequential(locations, method, Boolean.class,
@ -871,7 +872,7 @@ public class RouterClientProtocol implements ClientProtocol {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getFileLinkInfo",
new Class<?>[] {String.class}, new RemoteParam());
return rpcClient.invokeSequential(locations, method, HdfsFileStatus.class,
@ -883,7 +884,7 @@ public class RouterClientProtocol implements ClientProtocol {
boolean needBlockToken) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getLocatedFileInfo",
new Class<?>[] {String.class, boolean.class}, new RemoteParam(),
needBlockToken);
@ -1092,7 +1093,7 @@ public class RouterClientProtocol implements ClientProtocol {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(path, false);
rpcServer.getLocationsForPath(path, false, false);
RemoteMethod method = new RemoteMethod("listCorruptFileBlocks",
new Class<?>[] {String.class, String.class},
new RemoteParam(), cookie);
@ -1117,7 +1118,7 @@ public class RouterClientProtocol implements ClientProtocol {
// Get the summaries from regular files
final Collection<ContentSummary> summaries = new ArrayList<>();
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(path, false);
rpcServer.getLocationsForPath(path, false, false);
final RemoteMethod method = new RemoteMethod("getContentSummary",
new Class<?>[] {String.class}, new RemoteParam());
final List<RemoteResult<RemoteLocation, ContentSummary>> results =
@ -1169,7 +1170,7 @@ public class RouterClientProtocol implements ClientProtocol {
rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("fsync",
new Class<?>[] {String.class, long.class, String.class, long.class },
new RemoteParam(), fileId, clientName, lastBlockLength);
@ -1181,7 +1182,7 @@ public class RouterClientProtocol implements ClientProtocol {
rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("setTimes",
new Class<?>[] {String.class, long.class, long.class},
new RemoteParam(), mtime, atime);
@ -1211,7 +1212,7 @@ public class RouterClientProtocol implements ClientProtocol {
rpcServer.checkOperation(NameNode.OperationCategory.READ);
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(path, true);
rpcServer.getLocationsForPath(path, true, false);
RemoteMethod method = new RemoteMethod("getLinkTarget",
new Class<?>[] {String.class}, new RemoteParam());
return rpcClient.invokeSequential(locations, method, String.class, null);
@ -1309,7 +1310,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("modifyAclEntries",
new Class<?>[] {String.class, List.class},
new RemoteParam(), aclSpec);
@ -1327,7 +1328,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("removeAclEntries",
new Class<?>[] {String.class, List.class},
new RemoteParam(), aclSpec);
@ -1344,7 +1345,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("removeDefaultAcl",
new Class<?>[] {String.class}, new RemoteParam());
if (rpcServer.isInvokeConcurrent(src)) {
@ -1360,7 +1361,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("removeAcl",
new Class<?>[] {String.class}, new RemoteParam());
if (rpcServer.isInvokeConcurrent(src)) {
@ -1376,7 +1377,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod(
"setAcl", new Class<?>[] {String.class, List.class},
new RemoteParam(), aclSpec);
@ -1393,7 +1394,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getAclStatus",
new Class<?>[] {String.class}, new RemoteParam());
return rpcClient.invokeSequential(locations, method, AclStatus.class, null);
@ -1406,7 +1407,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("createEncryptionZone",
new Class<?>[] {String.class, String.class},
new RemoteParam(), keyName);
@ -1419,7 +1420,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getEZForPath",
new Class<?>[] {String.class}, new RemoteParam());
return rpcClient.invokeSequential(
@ -1453,7 +1454,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("setXAttr",
new Class<?>[] {String.class, XAttr.class, EnumSet.class},
new RemoteParam(), xAttr, flag);
@ -1472,7 +1473,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getXAttrs",
new Class<?>[] {String.class, List.class}, new RemoteParam(), xAttrs);
return (List<XAttr>) rpcClient.invokeSequential(
@ -1486,7 +1487,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, false);
rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("listXAttrs",
new Class<?>[] {String.class}, new RemoteParam());
return (List<XAttr>) rpcClient.invokeSequential(
@ -1499,7 +1500,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true);
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("removeXAttr",
new Class<?>[] {String.class, XAttr.class}, new RemoteParam(), xAttr);
if (rpcServer.isInvokeConcurrent(src)) {
@ -1515,7 +1516,7 @@ public class RouterClientProtocol implements ClientProtocol {
// TODO handle virtual directories
final List<RemoteLocation> locations =
rpcServer.getLocationsForPath(path, true);
rpcServer.getLocationsForPath(path, true, false);
RemoteMethod method = new RemoteMethod("checkAccess",
new Class<?>[] {String.class, FsAction.class},
new RemoteParam(), mode);
@ -1735,7 +1736,7 @@ public class RouterClientProtocol implements ClientProtocol {
throws IOException {
final List<RemoteLocation> dstLocations =
rpcServer.getLocationsForPath(dst, true);
rpcServer.getLocationsForPath(dst, true, false);
final Map<RemoteLocation, String> dstMap = new HashMap<>();
Iterator<RemoteLocation> iterator = srcLocations.iterator();
@ -2026,7 +2027,7 @@ public class RouterClientProtocol implements ClientProtocol {
try {
if (rpcServer.isPathAll(src)) {
List<RemoteLocation> locations;
locations = rpcServer.getLocationsForPath(src, false);
locations = rpcServer.getLocationsForPath(src, false, false);
RemoteMethod method = new RemoteMethod("getFileInfo",
new Class<?>[] {String.class}, new RemoteParam());
HdfsFileStatus fileStatus = rpcClient.invokeSequential(locations,

12
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterStoragePolicy.java
View File

@ -45,7 +45,8 @@ public class RouterStoragePolicy {
throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.WRITE);
List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, true);
List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("setStoragePolicy",
new Class<?>[] {String.class, String.class},
new RemoteParam(),
@ -67,7 +68,8 @@ public class RouterStoragePolicy {
public void unsetStoragePolicy(String src) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.WRITE, true);
List<RemoteLocation> locations = rpcServer.getLocationsForPath(src, true);
List<RemoteLocation> locations =
rpcServer.getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("unsetStoragePolicy",
new Class<?>[] {String.class},
new RemoteParam());
@ -82,7 +84,8 @@ public class RouterStoragePolicy {
throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ, true);
List<RemoteLocation> locations = rpcServer.getLocationsForPath(path, false);
List<RemoteLocation> locations =
rpcServer.getLocationsForPath(path, false, false);
RemoteMethod method = new RemoteMethod("getStoragePolicy",
new Class<?>[] {String.class},
new RemoteParam());
@ -92,7 +95,8 @@ public class RouterStoragePolicy {
public void satisfyStoragePolicy(String path) throws IOException {
rpcServer.checkOperation(NameNode.OperationCategory.READ, true);
List<RemoteLocation> locations = rpcServer.getLocationsForPath(path, true);
List<RemoteLocation> locations =
rpcServer.getLocationsForPath(path, true, false);
RemoteMethod method = new RemoteMethod("satisfyStoragePolicy",
new Class<?>[] {String.class},
new RemoteParam());

33
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNull;
@ -34,7 +35,9 @@ import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
@ -792,4 +795,34 @@ public class TestRouterQuota {
assertEquals(nsQuota, quota1.getQuota());
assertEquals(ssQuota, quota1.getSpaceQuota());
}
@Test
public void testNoQuotaaExceptionForUnrelatedOperations() throws Exception {
FileSystem nnFs = nnContext1.getFileSystem();
DistributedFileSystem routerFs =
(DistributedFileSystem) routerContext.getFileSystem();
Path path = new Path("/quota");
nnFs.mkdirs(new Path("/dir"));
MountTable mountTable1 = MountTable.newInstance("/quota",
Collections.singletonMap("ns0", "/dir"));
mountTable1.setQuota(new RouterQuotaUsage.Builder().quota(0).build());
addMountTable(mountTable1);
routerFs.mkdirs(new Path("/quota/1"));
routerContext.getRouter().getQuotaCacheUpdateService().periodicInvoke();
// Quota check for related operation.
intercept(NSQuotaExceededException.class,
"The NameSpace quota (directories and files) is exceeded",
() -> routerFs.mkdirs(new Path("/quota/2")));
//Quotas shouldn't be checked for unrelated operations.
routerFs.setStoragePolicy(path, "COLD");
routerFs.setErasureCodingPolicy(path, "RS-6-3-1024k");
routerFs.unsetErasureCodingPolicy(path);
routerFs.setPermission(path, new FsPermission((short) 01777));
routerFs.setOwner(path, "user", "group");
routerFs.setTimes(path, 1L, 1L);
routerFs.listStatus(path);
routerFs.getContentSummary(path);
}
}