From 33ffc960854b299573d4b0f449c34e08c42ee885 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Wed, 25 Apr 2018 15:22:26 +0800 Subject: [PATCH] HDFS-13484. RBF: Disable Nameservices from the federation. Contributed by Inigo Goiri. (cherry picked from commit 30fef0bf1e5c8c0ca073df99ad9b33cb0e4431a5) --- .../federation/metrics/FederationMetrics.java | 2 +- .../resolver/ActiveNamenodeResolver.java | 8 + .../FederationNamenodeServiceState.java | 3 +- .../resolver/MembershipNamenodeResolver.java | 112 ++++++--- .../federation/router/RouterAdminServer.java | 50 +++- .../router/RouterPermissionChecker.java | 59 ++++- .../federation/router/RouterRpcServer.java | 11 +- .../src/site/markdown/HDFSRouterFederation.md | 11 + .../federation/FederationTestUtils.java | 42 ++++ .../federation/MiniRouterDFSCluster.java | 8 + .../hdfs/server/federation/MockResolver.java | 6 + .../router/TestDisableNameservices.java | 236 ++++++++++++++++++ .../federation/router/TestRouterAdmin.java | 50 +++- .../federation/router/TestRouterAdminCLI.java | 11 + .../src/site/markdown/HDFSCommands.md | 4 + 15 files changed, 571 insertions(+), 42 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java index 39e060f984f..7f2cba22367 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java @@ -686,7 +686,7 @@ private List getActiveNamenodeRegistrations() namenodeResolver.getNamenodesForNameserviceId(nsId); if (nns != null) { FederationNamenodeContext nn = nns.get(0); - if (nn != null && nn instanceof MembershipState) { + if (nn instanceof MembershipState) { resultList.add((MembershipState) nn); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java index 1773b34925c..f1a53297d61 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java @@ -108,6 +108,14 @@ void updateActiveNamenode( */ Set getNamespaces() throws IOException; + /** + * Get a list of all namespaces that are disabled. + * + * @return List of name spaces identifier in the federation + * @throws IOException If the disabled list is not available. + */ + Set getDisabledNamespaces() throws IOException; + /** * Assign a unique identifier for the parent router service. * Required to report the status to the namenode resolver. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java index c773f820dc9..7907e3073e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java @@ -27,7 +27,8 @@ public enum FederationNamenodeServiceState { ACTIVE, // HAServiceState.ACTIVE or operational. STANDBY, // HAServiceState.STANDBY. UNAVAILABLE, // When the namenode cannot be reached. - EXPIRED; // When the last update is too old. + EXPIRED, // When the last update is too old. + DISABLED; // When the nameservice is disabled. public static FederationNamenodeServiceState getState(HAServiceState state) { switch(state) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java index 98ddd229402..0cdbdfd6c1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MembershipNamenodeResolver.java @@ -29,10 +29,13 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.server.federation.store.DisabledNameserviceStore; import org.apache.hadoop.hdfs.server.federation.store.MembershipStore; +import org.apache.hadoop.hdfs.server.federation.store.RecordStore; import org.apache.hadoop.hdfs.server.federation.store.StateStoreCache; import org.apache.hadoop.hdfs.server.federation.store.StateStoreService; import org.apache.hadoop.hdfs.server.federation.store.StateStoreUnavailableException; @@ -63,6 +66,8 @@ public class MembershipNamenodeResolver private final StateStoreService stateStore; /** Membership State Store interface. */ private MembershipStore membershipInterface; + /** Disabled Nameservice State Store interface. */ + private DisabledNameserviceStore disabledNameserviceInterface; /** Parent router ID. */ private String routerId; @@ -88,22 +93,38 @@ public MembershipNamenodeResolver( private synchronized MembershipStore getMembershipStore() throws IOException { if (this.membershipInterface == null) { - this.membershipInterface = this.stateStore.getRegisteredRecordStore( - MembershipStore.class); - if (this.membershipInterface == null) { - throw new IOException("State Store does not have an interface for " + - MembershipStore.class.getSimpleName()); - } + this.membershipInterface = getStoreInterface(MembershipStore.class); } return this.membershipInterface; } + private synchronized DisabledNameserviceStore getDisabledNameserviceStore() + throws IOException { + if (this.disabledNameserviceInterface == null) { + this.disabledNameserviceInterface = + getStoreInterface(DisabledNameserviceStore.class); + } + return this.disabledNameserviceInterface; + } + + private > T getStoreInterface(Class clazz) + throws IOException{ + T store = this.stateStore.getRegisteredRecordStore(clazz); + if (store == null) { + throw new IOException("State Store does not have an interface for " + + clazz.getSimpleName()); + } + return store; + } + @Override public boolean loadCache(boolean force) { // Our cache depends on the store, update it first try { MembershipStore membership = getMembershipStore(); membership.loadCache(force); + DisabledNameserviceStore disabled = getDisabledNameserviceStore(); + disabled.loadCache(force); } catch (IOException e) { LOG.error("Cannot update membership from the State Store", e); } @@ -151,30 +172,48 @@ public List getNamenodesForNameserviceId( final String nsId) throws IOException { List ret = cacheNS.get(nsId); - if (ret == null) { - try { - MembershipState partial = MembershipState.newInstance(); - partial.setNameserviceId(nsId); - GetNamenodeRegistrationsRequest request = - GetNamenodeRegistrationsRequest.newInstance(partial); - - final List result = - getRecentRegistrationForQuery(request, true, false); - if (result == null || result.isEmpty()) { - LOG.error("Cannot locate eligible NNs for {}", nsId); - return null; - } else { - cacheNS.put(nsId, result); - ret = result; - } - } catch (StateStoreUnavailableException e) { - LOG.error("Cannot get active NN for {}, State Store unavailable", nsId); - } + if (ret != null) { + return ret; } - if (ret == null) { + + // Not cached, generate the value + final List result; + try { + MembershipState partial = MembershipState.newInstance(); + partial.setNameserviceId(nsId); + GetNamenodeRegistrationsRequest request = + GetNamenodeRegistrationsRequest.newInstance(partial); + result = getRecentRegistrationForQuery(request, true, false); + } catch (StateStoreUnavailableException e) { + LOG.error("Cannot get active NN for {}, State Store unavailable", nsId); return null; } - return Collections.unmodifiableList(ret); + if (result == null || result.isEmpty()) { + LOG.error("Cannot locate eligible NNs for {}", nsId); + return null; + } + + // Mark disabled name services + try { + Set disabled = + getDisabledNameserviceStore().getDisabledNameservices(); + if (disabled == null) { + LOG.error("Cannot get disabled name services"); + } else { + for (MembershipState nn : result) { + if (disabled.contains(nn.getNameserviceId())) { + nn.setState(FederationNamenodeServiceState.DISABLED); + } + } + } + } catch (StateStoreUnavailableException e) { + LOG.error("Cannot get disabled name services, State Store unavailable"); + } + + // Cache the response + ret = Collections.unmodifiableList(result); + cacheNS.put(nsId, result); + return ret; } @Override @@ -260,7 +299,24 @@ public Set getNamespaces() throws IOException { GetNamespaceInfoRequest request = GetNamespaceInfoRequest.newInstance(); GetNamespaceInfoResponse response = getMembershipStore().getNamespaceInfo(request); - return response.getNamespaceInfo(); + Set nss = response.getNamespaceInfo(); + + // Filter disabled namespaces + Set ret = new TreeSet<>(); + Set disabled = getDisabledNamespaces(); + for (FederationNamespaceInfo ns : nss) { + if (!disabled.contains(ns.getNameserviceId())) { + ret.add(ns); + } + } + + return ret; + } + + @Override + public Set getDisabledNamespaces() throws IOException { + DisabledNameserviceStore store = getDisabledNameserviceStore(); + return store.getDisabledNameservices(); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java index da6779697fa..3da9a5a09b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java @@ -29,6 +29,8 @@ import org.apache.hadoop.hdfs.protocol.proto.RouterProtocolProtos.RouterAdminProtocolService; import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB; import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolServerSideTranslatorPB; +import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; +import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.store.DisabledNameserviceStore; import org.apache.hadoop.hdfs.server.federation.store.MountTableStore; @@ -282,28 +284,60 @@ private boolean verifySafeMode(boolean isInSafeMode) { @Override public DisableNameserviceResponse disableNameservice( DisableNameserviceRequest request) throws IOException { - // TODO check permissions + + RouterPermissionChecker pc = getPermissionChecker(); + if (pc != null) { + pc.checkSuperuserPrivilege(); + } + String nsId = request.getNameServiceId(); - // TODO check that the name service exists - boolean success = getDisabledNameserviceStore().disableNameservice(nsId); + boolean success = false; + if (namespaceExists(nsId)) { + success = getDisabledNameserviceStore().disableNameservice(nsId); + } else { + LOG.error("Cannot disable {}, it does not exists", nsId); + } return DisableNameserviceResponse.newInstance(success); } + private boolean namespaceExists(final String nsId) throws IOException { + boolean found = false; + ActiveNamenodeResolver resolver = router.getNamenodeResolver(); + Set nss = resolver.getNamespaces(); + for (FederationNamespaceInfo ns : nss) { + if (nsId.equals(ns.getNameserviceId())) { + found = true; + break; + } + } + return found; + } + @Override public EnableNameserviceResponse enableNameservice( EnableNameserviceRequest request) throws IOException { - // TODO check permissions + RouterPermissionChecker pc = getPermissionChecker(); + if (pc != null) { + pc.checkSuperuserPrivilege(); + } + String nsId = request.getNameServiceId(); - // TODO check that the name service exists - boolean success = getDisabledNameserviceStore().enableNameservice(nsId); + DisabledNameserviceStore store = getDisabledNameserviceStore(); + Set disabled = store.getDisabledNameservices(); + boolean success = false; + if (disabled.contains(nsId)) { + success = store.enableNameservice(nsId); + } else { + LOG.error("Cannot enable {}, it was not disabled", nsId); + } return EnableNameserviceResponse.newInstance(success); } @Override public GetDisabledNameservicesResponse getDisabledNameservices( GetDisabledNameservicesRequest request) throws IOException { - // TODO check permissions - Set nsIds = getDisabledNameserviceStore().getDisabledNameservices(); + Set nsIds = + getDisabledNameserviceStore().getDisabledNameservices(); return GetDisabledNameservicesResponse.newInstance(nsIds); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java index 9d81dce75bd..63d190c07cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterPermissionChecker.java @@ -17,12 +17,17 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; import org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker; +import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; @@ -35,9 +40,23 @@ public class RouterPermissionChecker extends FSPermissionChecker { /** Mount table default permission. */ public static final short MOUNT_TABLE_PERMISSION_DEFAULT = 00755; - public RouterPermissionChecker(String routerOwner, String supergroup, + /** Name of the super user. */ + private final String superUser; + /** Name of the super group. */ + private final String superGroup; + + public RouterPermissionChecker(String user, String group, UserGroupInformation callerUgi) { - super(routerOwner, supergroup, callerUgi, null); + super(user, group, callerUgi, null); + this.superUser = user; + this.superGroup = group; + } + + public RouterPermissionChecker(String user, String group) + throws IOException { + super(user, group, UserGroupInformation.getCurrentUser(), null); + this.superUser = user; + this.superGroup = group; } /** @@ -79,4 +98,40 @@ public void checkPermission(MountTable mountTable, FsAction access) + ": user " + getUser() + " does not have " + access.toString() + " permissions."); } + + /** + * Check the superuser privileges of the current RPC caller. This method is + * based on Datanode#checkSuperuserPrivilege(). + * @throws AccessControlException If the user is not authorized. + */ + @Override + public void checkSuperuserPrivilege() throws AccessControlException { + + // Try to get the ugi in the RPC call. + UserGroupInformation ugi = null; + try { + ugi = NameNode.getRemoteUser(); + } catch (IOException e) { + // Ignore as we catch it afterwards + } + if (ugi == null) { + LOG.error("Cannot get the remote user name"); + throw new AccessControlException("Cannot get the remote user name"); + } + + // Is this by the Router user itself? + if (ugi.getUserName().equals(superUser)) { + return; + } + + // Is the user a member of the super group? + List groups = Arrays.asList(ugi.getGroupNames()); + if (groups.contains(superGroup)) { + return; + } + + // Not a superuser + throw new AccessControlException( + ugi.getUserName() + " is not a super user"); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 2897823867c..b56ee5f0228 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.lang.reflect.Array; import java.net.InetSocketAddress; +import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; @@ -2138,7 +2139,15 @@ protected List getLocationsForPath(String path, } } - return location.getDestinations(); + // Filter disabled subclusters + Set disabled = namenodeResolver.getDisabledNamespaces(); + List locs = new ArrayList<>(); + for (RemoteLocation loc : location.getDestinations()) { + if (!disabled.contains(loc.getNameserviceId())) { + locs.add(loc); + } + } + return locs; } catch (IOException ioe) { if (this.rpcMonitor != null) { this.rpcMonitor.routerFailureStateStore(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md index fdaaa11627c..43e89ed6e1f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md @@ -229,6 +229,17 @@ Ls command will show below information for each mount table entry: Source Destinations Owner Group Mode Quota/Usage /path ns0->/path root supergroup rwxr-xr-x [NsQuota: 50/0, SsQuota: 100 B/0 B] +### Disabling nameservices + +To prevent accessing a nameservice (sublcuster), it can be disabled from the federation. +For example, one can disable `ns1`, list it and enable it again: + + [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -nameservice disable ns1 + [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -getDisabledNameservices + [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -nameservice enable ns1 + +This is useful when decommissioning subclusters or when one subcluster is missbehaving (e.g., low performance or unavailability). + Client configuration -------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java index b138e4d080b..ce320f462bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java @@ -20,6 +20,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; import java.io.BufferedReader; import java.io.FileNotFoundException; @@ -49,9 +52,18 @@ import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeContext; import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServiceState; import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; +import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.test.GenericTestUtils; +import org.mockito.internal.util.reflection.Whitebox; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.base.Supplier; @@ -60,6 +72,9 @@ */ public final class FederationTestUtils { + private static final Logger LOG = + LoggerFactory.getLogger(FederationTestUtils.class); + public final static String[] NAMESERVICES = {"ns0", "ns1"}; public final static String[] NAMENODES = {"nn0", "nn1", "nn2", "nn3"}; public final static String[] ROUTERS = @@ -274,4 +289,31 @@ public static boolean deleteFile(FileSystem fs, String path) throws IOException { return fs.delete(new Path(path), true); } + + /** + * Simulate that a Namenode is slow by adding a sleep to the check operation + * in the NN. + * @param nn Namenode to simulate slow. + * @param seconds Number of seconds to add to the Namenode. + * @throws Exception If we cannot add the sleep time. + */ + public static void simulateSlowNamenode(final NameNode nn, final int seconds) + throws Exception { + FSNamesystem namesystem = nn.getNamesystem(); + HAContext haContext = namesystem.getHAContext(); + HAContext spyHAContext = spy(haContext); + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + LOG.info("Simulating slow namenode {}", invocation.getMock()); + try { + Thread.sleep(seconds * 1000); + } catch(InterruptedException e) { + LOG.error("Simulating a slow namenode aborted"); + } + return null; + } + }).when(spyHAContext).checkOperation(any(OperationCategory.class)); + Whitebox.setInternalState(namesystem, "haContext", spyHAContext); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java index 0a4de33f835..e34713d665a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MiniRouterDFSCluster.java @@ -185,6 +185,10 @@ public FileContext getFileContext() { return this.fileContext; } + public URI getFileSystemURI() { + return fileSystemUri; + } + public String getHttpAddress() { InetSocketAddress httpAddress = router.getHttpServerAddress(); return NetUtils.getHostPortString(httpAddress); @@ -236,6 +240,10 @@ public RouterClient getAdminClient() throws IOException { return adminClient; } + public void resetAdminClient() { + adminClient = null; + } + public DFSClient getClient() throws IOException, URISyntaxException { if (client == null) { LOG.info("Connecting to router at {}", fileSystemUri); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java index 0ce0944854b..36cce391aea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; @@ -262,6 +263,11 @@ public Set getNamespaces() throws IOException { return this.namespaces; } + @Override + public Set getDisabledNamespaces() throws IOException { + return new TreeSet<>(); + } + @Override public PathLocation getDestinationForPath(String path) throws IOException { List remoteLocations = new LinkedList<>(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java new file mode 100644 index 00000000000..15b104df215 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDisableNameservices.java @@ -0,0 +1,236 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.simulateSlowNamenode; +import static org.apache.hadoop.util.Time.monotonicNow; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; +import org.apache.hadoop.hdfs.server.federation.metrics.FederationMetrics; +import org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver; +import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; +import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver; +import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder; +import org.apache.hadoop.hdfs.server.federation.store.DisabledNameserviceStore; +import org.apache.hadoop.hdfs.server.federation.store.StateStoreService; +import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.DisableNameserviceRequest; +import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.codehaus.jettison.json.JSONObject; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Test the behavior when disabling name services. + */ +public class TestDisableNameservices { + + private static StateStoreDFSCluster cluster; + private static RouterContext routerContext; + private static RouterClient routerAdminClient; + private static ClientProtocol routerProtocol; + + @BeforeClass + public static void setUp() throws Exception { + // Build and start a federated cluster + cluster = new StateStoreDFSCluster(false, 2); + Configuration routerConf = new RouterConfigBuilder() + .stateStore() + .metrics() + .admin() + .rpc() + .build(); + // Reduce the number of RPC threads to saturate the Router easy + routerConf.setInt(RBFConfigKeys.DFS_ROUTER_HANDLER_COUNT_KEY, 8); + routerConf.setInt(RBFConfigKeys.DFS_ROUTER_CLIENT_THREADS_SIZE, 4); + + // Set the DNs to belong to only one subcluster + cluster.setIndependentDNs(); + + cluster.addRouterOverrides(routerConf); + // override some settings for the client + cluster.startCluster(); + cluster.startRouters(); + cluster.waitClusterUp(); + + routerContext = cluster.getRandomRouter(); + routerProtocol = routerContext.getClient().getNamenode(); + routerAdminClient = routerContext.getAdminClient(); + + setupNamespace(); + + // Simulate one of the subclusters to be slow + MiniDFSCluster dfsCluster = cluster.getCluster(); + NameNode nn0 = dfsCluster.getNameNode(0); + simulateSlowNamenode(nn0, 1); + } + + private static void setupNamespace() throws IOException { + + // Setup a mount table to map to the two namespaces + MountTableManager mountTable = routerAdminClient.getMountTableManager(); + Map destinations = new TreeMap<>(); + destinations.put("ns0", "/"); + destinations.put("ns1", "/"); + MountTable newEntry = MountTable.newInstance("/", destinations); + newEntry.setDestOrder(DestinationOrder.RANDOM); + AddMountTableEntryRequest request = + AddMountTableEntryRequest.newInstance(newEntry); + mountTable.addMountTableEntry(request); + + // Refresh the cache in the Router + Router router = routerContext.getRouter(); + MountTableResolver mountTableResolver = + (MountTableResolver) router.getSubclusterResolver(); + mountTableResolver.loadCache(true); + + // Add a folder to each namespace + NamenodeContext nn0 = cluster.getNamenode("ns0", null); + nn0.getFileSystem().mkdirs(new Path("/dirns0")); + NamenodeContext nn1 = cluster.getNamenode("ns1", null); + nn1.getFileSystem().mkdirs(new Path("/dirns1")); + } + + @AfterClass + public static void tearDown() { + if (cluster != null) { + cluster.stopRouter(routerContext); + cluster.shutdown(); + cluster = null; + } + } + + @After + public void cleanup() throws IOException { + Router router = routerContext.getRouter(); + StateStoreService stateStore = router.getStateStore(); + DisabledNameserviceStore store = + stateStore.getRegisteredRecordStore(DisabledNameserviceStore.class); + store.loadCache(true); + + Set disabled = store.getDisabledNameservices(); + for (String nsId : disabled) { + store.enableNameservice(nsId); + } + store.loadCache(true); + } + + @Test + public void testWithoutDisabling() throws IOException { + + // ns0 is slow and renewLease should take a long time + long t0 = monotonicNow(); + routerProtocol.renewLease("client0"); + long t = monotonicNow() - t0; + assertTrue("It took too little: " + t + "ms", + t > TimeUnit.SECONDS.toMillis(1)); + + // Return the results from all subclusters even if slow + FileSystem routerFs = routerContext.getFileSystem(); + FileStatus[] filesStatus = routerFs.listStatus(new Path("/")); + assertEquals(2, filesStatus.length); + assertEquals("dirns0", filesStatus[0].getPath().getName()); + assertEquals("dirns1", filesStatus[1].getPath().getName()); + } + + @Test + public void testDisabling() throws Exception { + + disableNameservice("ns0"); + + // renewLease should be fast as we are skipping ns0 + long t0 = monotonicNow(); + routerProtocol.renewLease("client0"); + long t = monotonicNow() - t0; + assertTrue("It took too long: " + t + "ms", + t < TimeUnit.SECONDS.toMillis(1)); + + // We should not report anything from ns0 + FileSystem routerFs = routerContext.getFileSystem(); + FileStatus[] filesStatus = routerFs.listStatus(new Path("/")); + assertEquals(1, filesStatus.length); + assertEquals("dirns1", filesStatus[0].getPath().getName()); + } + + @Test + public void testMetrics() throws Exception { + disableNameservice("ns0"); + + int numActive = 0; + int numDisabled = 0; + Router router = routerContext.getRouter(); + FederationMetrics metrics = router.getMetrics(); + String jsonString = metrics.getNameservices(); + JSONObject jsonObject = new JSONObject(jsonString); + Iterator keys = jsonObject.keys(); + while (keys.hasNext()) { + String key = (String) keys.next(); + JSONObject json = jsonObject.getJSONObject(key); + String nsId = json.getString("nameserviceId"); + String state = json.getString("state"); + if (nsId.equals("ns0")) { + assertEquals("DISABLED", state); + numDisabled++; + } else { + assertEquals("ACTIVE", state); + numActive++; + } + } + assertEquals(1, numActive); + assertEquals(1, numDisabled); + } + + private static void disableNameservice(final String nsId) + throws IOException { + NameserviceManager nsManager = routerAdminClient.getNameserviceManager(); + DisableNameserviceRequest req = + DisableNameserviceRequest.newInstance(nsId); + nsManager.disableNameservice(req); + + Router router = routerContext.getRouter(); + StateStoreService stateStore = router.getStateStore(); + DisabledNameserviceStore store = + stateStore.getRegisteredRecordStore(DisabledNameserviceStore.class); + store.loadCache(true); + MembershipNamenodeResolver resolver = + (MembershipNamenodeResolver) router.getNamenodeResolver(); + resolver.loadCache(true); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java index 5e271731aa0..769bfe71ed8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java @@ -17,21 +17,27 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createNamenodeReport; import static org.apache.hadoop.hdfs.server.federation.store.FederationStateStoreTestUtils.synchronizeRecords; +import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.Collections; import java.util.List; import java.util.Set; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; +import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder; @@ -52,6 +58,7 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.records.MountTable; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.junit.AfterClass; import org.junit.Before; @@ -86,6 +93,14 @@ public static void globalSetUp() throws Exception { mockMountTable = cluster.generateMockMountTable(); Router router = routerContext.getRouter(); stateStore = router.getStateStore(); + + // Add two name services for testing disabling + ActiveNamenodeResolver membership = router.getNamenodeResolver(); + membership.registerNamenode( + createNamenodeReport("ns0", "nn1", HAServiceState.ACTIVE)); + membership.registerNamenode( + createNamenodeReport("ns1", "nn1", HAServiceState.ACTIVE)); + stateStore.refreshCaches(true); } @AfterClass @@ -97,6 +112,8 @@ public static void tearDown() { public void testSetup() throws Exception { assertTrue( synchronizeRecords(stateStore, mockMountTable, MountTable.class)); + // Avoid running with random users + routerContext.resetAdminClient(); } @Test @@ -375,6 +392,37 @@ public void testNameserviceManager() throws IOException { assertTrue(enableResp.getStatus()); disabled = getDisabledNameservices(nsManager); assertTrue(disabled.isEmpty()); + + // Non existing name services should fail + disableReq = DisableNameserviceRequest.newInstance("nsunknown"); + disableResp = nsManager.disableNameservice(disableReq); + assertFalse(disableResp.getStatus()); + } + + @Test + public void testNameserviceManagerUnauthorized() throws Exception { + + // Try to disable a name service with a random user + final String username = "baduser"; + UserGroupInformation user = + UserGroupInformation.createRemoteUser(username); + user.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + RouterClient client = routerContext.getAdminClient(); + NameserviceManager nameservices = client.getNameserviceManager(); + DisableNameserviceRequest disableReq = + DisableNameserviceRequest.newInstance("ns0"); + try { + nameservices.disableNameservice(disableReq); + fail("We should not be able to disable nameservices"); + } catch (IOException ioe) { + assertExceptionContains( + username + " is not a super user", ioe); + } + return null; + } + }); } private Set getDisabledNameservices(NameserviceManager nsManager) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java index cd5edf01eed..1ff07ac340e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.federation.router; +import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createNamenodeReport; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -27,10 +28,12 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext; import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster; +import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder; @@ -93,6 +96,14 @@ public static void globalSetUp() throws Exception { routerSocket); admin = new RouterAdmin(routerConf); client = routerContext.getAdminClient(); + + // Add two fake name services to testing disabling them + ActiveNamenodeResolver membership = router.getNamenodeResolver(); + membership.registerNamenode( + createNamenodeReport("ns0", "nn1", HAServiceState.ACTIVE)); + membership.registerNamenode( + createNamenodeReport("ns1", "nn1", HAServiceState.ACTIVE)); + stateStore.refreshCaches(true); } @AfterClass diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index 0bada59c673..cb5d8a71fbf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -415,6 +415,8 @@ Usage: [-setQuota -nsQuota -ssQuota ] [-clrQuota ] [-safemode enter | leave | get] + [-nameservice disable | enable ] + [-getDisabledNameservices] | COMMAND\_OPTION | Description | |:---- |:---- | @@ -424,6 +426,8 @@ Usage: | `-setQuota` *path* `-nsQuota` *nsQuota* `-ssQuota` *ssQuota* | Set quota for specified path. See [HDFS Quotas Guide](./HdfsQuotaAdminGuide.html) for the quota detail. | | `-clrQuota` *path* | Clear quota of given mount point. See [HDFS Quotas Guide](./HdfsQuotaAdminGuide.html) for the quota detail. | | `-safemode` `enter` `leave` `get` | Manually set the Router entering or leaving safe mode. The option *get* will be used for verifying if the Router is in safe mode state. | +| `-nameservice` `disable` `enable` *nameservice* | Disable/enable a name service from the federation. If disabled, requests will not go to that name service. | +| `-getDisabledNameservices` | Get the name services that are disabled in the federation. | The commands for managing Router-based federation. See [Mount table management](./HDFSRouterFederation.html#Mount_table_management) for more info.