From b83ff77409a5f9b1ae2fff3dc5ab956df398c4c9 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Fri, 11 May 2018 18:53:12 +0800 Subject: [PATCH] Revert "HDFS-13346. RBF: Fix synchronization of router quota and nameservice quota." This reverts commit d4a141e7b014f23d2b3bb1aa215a8799fdc5315e. --- .../federation/router/RouterAdminServer.java | 28 +----------- .../federation/router/TestRouterAdminCLI.java | 10 ----- .../federation/router/TestRouterQuota.java | 43 +------------------ 3 files changed, 2 insertions(+), 79 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java index 139dfb82fe2..3da9a5a09b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java @@ -26,7 +26,6 @@ import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.proto.RouterProtocolProtos.RouterAdminProtocolService; import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB; import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolServerSideTranslatorPB; @@ -55,7 +54,6 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableE import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse; -import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; @@ -230,31 +228,7 @@ public class RouterAdminServer extends AbstractService @Override public UpdateMountTableEntryResponse updateMountTableEntry( UpdateMountTableEntryRequest request) throws IOException { - UpdateMountTableEntryResponse response = - getMountTableStore().updateMountTableEntry(request); - - MountTable mountTable = request.getEntry(); - if (mountTable != null) { - synchronizeQuota(mountTable); - } - return response; - } - - /** - * Synchronize the quota value across mount table and subclusters. - * @param mountTable Quota set in given mount table. - * @throws IOException - */ - private void synchronizeQuota(MountTable mountTable) throws IOException { - String path = mountTable.getSourcePath(); - long nsQuota = mountTable.getQuota().getQuota(); - long ssQuota = mountTable.getQuota().getSpaceQuota(); - - if (nsQuota != HdfsConstants.QUOTA_DONT_SET - || ssQuota != HdfsConstants.QUOTA_DONT_SET) { - this.router.getRpcServer().getQuotaModule().setQuota(path, nsQuota, - ssQuota, null); - } + return getMountTableStore().updateMountTableEntry(request); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java index 7e04e617fbf..2537c193c04 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java @@ -51,8 +51,6 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.Mockito; -import org.mockito.internal.util.reflection.Whitebox; import com.google.common.base.Supplier; @@ -106,14 +104,6 @@ public class TestRouterAdminCLI { membership.registerNamenode( createNamenodeReport("ns1", "nn1", HAServiceState.ACTIVE)); stateStore.refreshCaches(true); - - // Mock the quota module since no real namenode is started up. - Quota quota = Mockito - .spy(routerContext.getRouter().createRpcServer().getQuotaModule()); - Mockito.doNothing().when(quota).setQuota(Mockito.anyString(), - Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); - Whitebox.setInternalState( - routerContext.getRouter().getRpcServer(), "quotaCall", quota); } @AfterClass diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java index 3a713cad248..3183df4749e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.server.federation.router; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNull; import java.io.IOException; @@ -38,9 +37,9 @@ import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; -import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver; @@ -50,10 +49,8 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntr import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse; -import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.util.Time; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -455,42 +452,4 @@ public class TestRouterQuota { return removeResponse.getEntries(); } - - @Test - public void testQuotaSynchronization() throws IOException { - long updateNsQuota = 3; - long updateSsQuota = 4; - MountTable mountTable = MountTable.newInstance("/quotaSync", - Collections.singletonMap("ns0", "/"), Time.now(), Time.now()); - mountTable.setQuota(new RouterQuotaUsage.Builder().quota(1) - .spaceQuota(2).build()); - // Add new mount table - addMountTable(mountTable); - - // ensure the quota is not set as updated value - QuotaUsage realQuota = nnContext1.getFileSystem() - .getQuotaUsage(new Path("/")); - assertNotEquals(updateNsQuota, realQuota.getQuota()); - assertNotEquals(updateSsQuota, realQuota.getSpaceQuota()); - - // Call periodicInvoke to ensure quota updated in quota manager - // and state store. - RouterQuotaUpdateService updateService = routerContext.getRouter() - .getQuotaCacheUpdateService(); - updateService.periodicInvoke(); - - mountTable.setQuota(new RouterQuotaUsage.Builder().quota(updateNsQuota) - .spaceQuota(updateSsQuota).build()); - UpdateMountTableEntryRequest updateRequest = UpdateMountTableEntryRequest - .newInstance(mountTable); - RouterClient client = routerContext.getAdminClient(); - MountTableManager mountTableManager = client.getMountTableManager(); - mountTableManager.updateMountTableEntry(updateRequest); - - // verify if the quota is updated in real path - realQuota = nnContext1.getFileSystem().getQuotaUsage( - new Path("/")); - assertEquals(updateNsQuota, realQuota.getQuota()); - assertEquals(updateSsQuota, realQuota.getSpaceQuota()); - } }