diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java index d885989efa2..07c8a69f246 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocol.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.protocolPB; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; +import org.apache.hadoop.hdfs.server.federation.resolver.RouterGenericManager; import org.apache.hadoop.hdfs.server.federation.router.NameserviceManager; import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager; import org.apache.hadoop.ipc.GenericRefreshProtocol; @@ -30,5 +31,6 @@ import org.apache.hadoop.ipc.GenericRefreshProtocol; @InterfaceAudience.Private @InterfaceStability.Stable public interface RouterAdminProtocol extends MountTableManager, - RouterStateManager, NameserviceManager, GenericRefreshProtocol { + RouterStateManager, NameserviceManager, GenericRefreshProtocol, + RouterGenericManager { } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java index 6f6724e7382..861df81fcc2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java @@ -45,6 +45,8 @@ import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProt import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationResponseProto; import org.apache.hadoop.hdfs.server.federation.router.RouterAdminServer; import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse; @@ -54,6 +56,7 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.EnableNameservice import org.apache.hadoop.hdfs.server.federation.store.protocol.EnableNameserviceResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshSuperUserGroupsConfigurationResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.GetDisabledNameservicesRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.GetDisabledNameservicesResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.GetDestinationRequest; @@ -90,6 +93,7 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafe import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshMountTableEntriesRequestPBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshMountTableEntriesResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshSuperUserGroupsConfigurationResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl; @@ -197,6 +201,28 @@ public class RouterAdminProtocolServerSideTranslatorPB implements } } + /** + * Refresh superuser proxy groups mappings. + */ + @Override + public RefreshSuperUserGroupsConfigurationResponseProto + refreshSuperUserGroupsConfiguration( + RpcController controller, + RefreshSuperUserGroupsConfigurationRequestProto request) + throws ServiceException { + try { + boolean result = server.refreshSuperUserGroupsConfiguration(); + RefreshSuperUserGroupsConfigurationResponse response = + RefreshSuperUserGroupsConfigurationResponsePBImpl.newInstance(result); + RefreshSuperUserGroupsConfigurationResponsePBImpl responsePB = + (RefreshSuperUserGroupsConfigurationResponsePBImpl) response; + return responsePB.getProto(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override public EnterSafeModeResponseProto enterSafeMode(RpcController controller, EnterSafeModeRequestProto request) throws ServiceException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java index 9cdc3c1c940..c4cf73bdbd4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java @@ -42,11 +42,14 @@ import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProt import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshMountTableEntriesRequestProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshMountTableEntriesResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationResponseProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; +import org.apache.hadoop.hdfs.server.federation.resolver.RouterGenericManager; import org.apache.hadoop.hdfs.server.federation.router.NameserviceManager; import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager; import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest; @@ -89,6 +92,7 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetSafeMo import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshMountTableEntriesRequestPBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshMountTableEntriesResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshSuperUserGroupsConfigurationResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl; @@ -110,7 +114,8 @@ import com.google.protobuf.ServiceException; @InterfaceStability.Stable public class RouterAdminProtocolTranslatorPB implements ProtocolMetaInterface, MountTableManager, - Closeable, ProtocolTranslator, RouterStateManager, NameserviceManager { + Closeable, ProtocolTranslator, RouterStateManager, NameserviceManager, + RouterGenericManager { final private RouterAdminProtocolPB rpcProxy; public RouterAdminProtocolTranslatorPB(RouterAdminProtocolPB proxy) { @@ -309,4 +314,18 @@ public class RouterAdminProtocolTranslatorPB throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); } } + + @Override + public boolean refreshSuperUserGroupsConfiguration() throws IOException { + RefreshSuperUserGroupsConfigurationRequestProto proto = + RefreshSuperUserGroupsConfigurationRequestProto.newBuilder().build(); + try { + RefreshSuperUserGroupsConfigurationResponseProto response = + rpcProxy.refreshSuperUserGroupsConfiguration(null, proto); + return new RefreshSuperUserGroupsConfigurationResponsePBImpl(response) + .getStatus(); + } catch (ServiceException e) { + throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RouterGenericManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RouterGenericManager.java new file mode 100644 index 00000000000..efd898a5577 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/RouterGenericManager.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.resolver; + +import java.io.IOException; + +/** + * Generic methods for managing Router. + */ +public interface RouterGenericManager { + /** + * Refresh superuser proxy groups mappings (used in RBF). + * @return true if the operation was successful. + * @throws IOException if operation was not successful. + */ + boolean refreshSuperUserGroupsConfiguration() throws IOException; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java index 5975cec6fee..cd58926143a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java @@ -84,6 +84,7 @@ import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolPB; import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolServerSideTranslatorPB; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.AbstractService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -619,4 +620,10 @@ public class RouterAdminServer extends AbstractService // Let the registry handle as needed return RefreshRegistry.defaultRegistry().dispatch(identifier, args); } + + @Override // RouterGenericManager + public boolean refreshSuperUserGroupsConfiguration() throws IOException { + ProxyUsers.refreshSuperUserGroupsConfiguration(); + return true; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java index 95fe1a8d437..0641c0b82af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClient.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolPB; import org.apache.hadoop.hdfs.protocolPB.RouterAdminProtocolTranslatorPB; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; +import org.apache.hadoop.hdfs.server.federation.resolver.RouterGenericManager; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; @@ -77,6 +78,10 @@ public class RouterClient implements Closeable { return proxy; } + public RouterGenericManager getRouterGenericManager() { + return proxy; + } + @Override public synchronized void close() throws IOException { RPC.stopProxy(proxy); diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshSuperUserGroupsConfigurationRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshSuperUserGroupsConfigurationRequest.java new file mode 100644 index 00000000000..8cc8e89bc48 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshSuperUserGroupsConfigurationRequest.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.store.protocol; + +import java.io.IOException; + +import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer; + +/** + * API request for refreshing super user groups on router. + */ +public abstract class RefreshSuperUserGroupsConfigurationRequest { + public static RefreshSuperUserGroupsConfigurationRequest newInstance() + throws IOException { + return StateStoreSerializer + .newRecord(RefreshSuperUserGroupsConfigurationRequest.class); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshSuperUserGroupsConfigurationResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshSuperUserGroupsConfigurationResponse.java new file mode 100644 index 00000000000..11d265d6707 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RefreshSuperUserGroupsConfigurationResponse.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.store.protocol; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer; + +/** + * API response for refreshing super user groups on router. + */ +public abstract class RefreshSuperUserGroupsConfigurationResponse { + + public static RefreshSuperUserGroupsConfigurationResponse newInstance() + throws IOException { + return StateStoreSerializer. + newRecord(RefreshSuperUserGroupsConfigurationResponse.class); + } + + public static RefreshSuperUserGroupsConfigurationResponse + newInstance(boolean status) throws IOException { + RefreshSuperUserGroupsConfigurationResponse response = newInstance(); + response.setStatus(status); + return response; + } + + @Public + @Unstable + public abstract boolean getStatus(); + + @Public + @Unstable + public abstract void setStatus(boolean result); +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RefreshSuperUserGroupsConfigurationRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RefreshSuperUserGroupsConfigurationRequestPBImpl.java new file mode 100644 index 00000000000..00573fe976a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RefreshSuperUserGroupsConfigurationRequestPBImpl.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb; + +import com.google.protobuf.Message; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationRequestProtoOrBuilder; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationRequestProto.Builder; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshSuperUserGroupsConfigurationRequest; +import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord; + +import java.io.IOException; + +/** + * Protobuf implementation of the state store API object + * RefreshSuperUserGroupsConfigurationRequest. + */ +public class RefreshSuperUserGroupsConfigurationRequestPBImpl + extends RefreshSuperUserGroupsConfigurationRequest + implements PBRecord { + + private FederationProtocolPBTranslator< + RefreshSuperUserGroupsConfigurationRequestProto, Builder, + RefreshSuperUserGroupsConfigurationRequestProtoOrBuilder> translator = + new FederationProtocolPBTranslator<>( + RefreshSuperUserGroupsConfigurationRequestProto.class); + + public RefreshSuperUserGroupsConfigurationRequestPBImpl() { + } + + public RefreshSuperUserGroupsConfigurationRequestPBImpl( + RefreshSuperUserGroupsConfigurationRequestProto proto) { + this.translator.setProto(proto); + } + + @Override + public RefreshSuperUserGroupsConfigurationRequestProto getProto() { + this.translator.getBuilder(); + return this.translator.build(); + } + + @Override + public void setProto(Message proto) { + this.translator.setProto(proto); + } + + @Override + public void readInstance(String base64String) throws IOException { + this.translator.readInstance(base64String); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RefreshSuperUserGroupsConfigurationResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RefreshSuperUserGroupsConfigurationResponsePBImpl.java new file mode 100644 index 00000000000..47ae3be62ee --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RefreshSuperUserGroupsConfigurationResponsePBImpl.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb; + +import com.google.protobuf.Message; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationResponseProtoOrBuilder; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationResponseProto.Builder; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshSuperUserGroupsConfigurationResponse; +import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord; + +import java.io.IOException; + +/** + * Protobuf implementation of the state store API object + * RefreshSuperUserGroupsConfigurationResponse. + */ +public class RefreshSuperUserGroupsConfigurationResponsePBImpl + extends RefreshSuperUserGroupsConfigurationResponse + implements PBRecord { + + private FederationProtocolPBTranslator< + RefreshSuperUserGroupsConfigurationResponseProto, + Builder, + RefreshSuperUserGroupsConfigurationResponseProtoOrBuilder> translator = + new FederationProtocolPBTranslator<>( + RefreshSuperUserGroupsConfigurationResponseProto.class); + + public RefreshSuperUserGroupsConfigurationResponsePBImpl() { + } + + public RefreshSuperUserGroupsConfigurationResponsePBImpl( + RefreshSuperUserGroupsConfigurationResponseProto proto) { + this.translator.setProto(proto); + } + + @Override + public RefreshSuperUserGroupsConfigurationResponseProto getProto() { + return translator.build(); + } + + @Override + public void setProto(Message proto) { + this.translator.setProto(proto); + } + + @Override + public void readInstance(String base64String) throws IOException { + this.translator.readInstance(base64String); + } + + @Override + public boolean getStatus() { + return this.translator.getProtoOrBuilder().getStatus(); + } + + @Override + public void setStatus(boolean result) { + this.translator.getBuilder().setStatus(result); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java index 4f8dcdeaa84..1f050ad1004 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; +import org.apache.hadoop.hdfs.server.federation.resolver.RouterGenericManager; import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder; import org.apache.hadoop.hdfs.server.federation.router.NameserviceManager; import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys; @@ -127,7 +128,8 @@ public class RouterAdmin extends Configured implements Tool { {"-add", "-update", "-rm", "-ls", "-getDestination", "-setQuota", "-clrQuota", "-safemode", "-nameservice", "-getDisabledNameservices", - "-refresh", "-refreshRouterArgs"}; + "-refresh", "-refreshRouterArgs", + "-refreshSuperUserGroupsConfiguration"}; StringBuilder usage = new StringBuilder(); usage.append("Usage: hdfs dfsrouteradmin :\n"); for (int i = 0; i < commands.length; i++) { @@ -170,6 +172,8 @@ public class RouterAdmin extends Configured implements Tool { return "\t[-refresh]"; } else if (cmd.equals("-refreshRouterArgs")) { return "\t[-refreshRouterArgs [arg1..argn]]"; + } else if (cmd.equals("-refreshSuperUserGroupsConfiguration")) { + return "\t[-refreshSuperUserGroupsConfiguration]"; } return getUsage(null); } @@ -203,6 +207,10 @@ public class RouterAdmin extends Configured implements Tool { if (arg.length > 1) { throw new IllegalArgumentException("No arguments allowed"); } + } else if (arg[0].equals("-refreshSuperUserGroupsConfiguration")) { + if (arg.length > 1) { + throw new IllegalArgumentException("No arguments allowed"); + } } } @@ -348,6 +356,8 @@ public class RouterAdmin extends Configured implements Tool { refresh(address); } else if ("-refreshRouterArgs".equals(cmd)) { exitCode = genericRefresh(argv, i); + } else if ("-refreshSuperUserGroupsConfiguration".equals(cmd)) { + exitCode = refreshSuperUserGroupsConfiguration(); } else { throw new IllegalArgumentException("Unknown Command: " + cmd); } @@ -387,6 +397,25 @@ public class RouterAdmin extends Configured implements Tool { return exitCode; } + /** + * Refresh superuser proxy groups mappings on Router. + * + * @throws IOException if the operation was not successful. + */ + private int refreshSuperUserGroupsConfiguration() + throws IOException{ + RouterGenericManager proxy = client.getRouterGenericManager(); + String address = getConf().getTrimmed( + RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY, + RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_DEFAULT); + if(proxy.refreshSuperUserGroupsConfiguration()){ + System.out.println( + "Successfully updated superuser proxy groups on router " + address); + return 0; + } + return -1; + } + private void refresh(String address) throws IOException { if (refreshRouterCache()) { System.out.println( diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto index 35566ae393f..5b6dfd9c965 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto @@ -242,6 +242,13 @@ message RefreshMountTableEntriesResponseProto { optional bool result = 1; } +message RefreshSuperUserGroupsConfigurationRequestProto { +} + +message RefreshSuperUserGroupsConfigurationResponseProto { + optional bool status = 1; +} + ///////////////////////////////////////////////// // Route State ///////////////////////////////////////////////// diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto index e8fc2683e00..18bad54274b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto @@ -84,4 +84,9 @@ service RouterAdminProtocolService { * Get the destination of a file/directory in the federation. */ rpc getDestination(GetDestinationRequestProto) returns (GetDestinationResponseProto); + + /** + * Refresh superuser proxy groups mappings on Router. + */ + rpc refreshSuperUserGroupsConfiguration(RefreshSuperUserGroupsConfigurationRequestProto) returns (RefreshSuperUserGroupsConfigurationResponseProto); } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRefreshSuperUserGroupsConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRefreshSuperUserGroupsConfiguration.java new file mode 100644 index 00000000000..fb88882243f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRefreshSuperUserGroupsConfiguration.java @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.router; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster; +import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder; +import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; +import org.apache.hadoop.hdfs.tools.federation.RouterAdmin; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.authorize.ProxyUsers; + +import org.apache.hadoop.test.LambdaTestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.net.URL; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test refreshSuperUserGroupsConfiguration on Routers. + * Notice that this test is intended to test + * {@link RouterAdminServer#refreshSuperUserGroupsConfiguration} + * which invoked by {@link RouterAdmin} + */ +public class TestRouterRefreshSuperUserGroupsConfiguration { + private static final Logger LOG = LoggerFactory.getLogger( + TestRouterRefreshSuperUserGroupsConfiguration.class); + + private MiniRouterDFSCluster cluster; + private static final String ROUTER_NS = "rbfns"; + private static final String HDFS_SCHEMA = "hdfs://"; + private static final String LOOPBACK_ADDRESS = "127.0.0.1"; + + private String tempResource = null; + @Before + public void setUpCluster() throws Exception { + Configuration conf = new RouterConfigBuilder() + .rpc() + .admin() + .build(); + cluster = new MiniRouterDFSCluster(false, 1); + cluster.addRouterOverrides(conf); + cluster.startRouters(); + } + + @After + public void tearDown() { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + + if (tempResource != null) { + File f = new File(tempResource); + f.delete(); + tempResource = null; + } + } + + private Configuration initializeClientConfig() throws Exception { + + Configuration conf = new Configuration(false); + Router router = cluster.getRouters().get(0).getRouter(); + + conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns0,ns1,"+ ROUTER_NS); + conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "."+ ROUTER_NS, "r1"); + conf.set(RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY, + LOOPBACK_ADDRESS + ":" + router.getAdminServerAddress().getPort()); + conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + + "." + ROUTER_NS + ".r1", + LOOPBACK_ADDRESS + ":" + router.getRpcServerAddress().getPort()); + conf.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + + "." + ROUTER_NS, + ConfiguredFailoverProxyProvider.class.getCanonicalName()); + conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, + HDFS_SCHEMA + ROUTER_NS); + conf.setBoolean("dfs.client.failover.random.order", false); + conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, HDFS_SCHEMA + ROUTER_NS); + + return conf; + } + + @Test + public void testRefreshSuperUserGroupsConfiguration() + throws Exception { + Configuration conf = initializeClientConfig(); + testRefreshSuperUserGroupsConfigurationInternal(conf); + } + + private void testRefreshSuperUserGroupsConfigurationInternal( + Configuration conf) throws Exception { + + UserGroupInformation ugi = mock(UserGroupInformation.class); + UserGroupInformation impersonator = mock(UserGroupInformation.class); + + // Setting for impersonator + when(impersonator.getShortUserName()).thenReturn("impersonator"); + when(impersonator.getUserName()).thenReturn("impersonator"); + + // Setting for victim + when(ugi.getRealUser()).thenReturn(impersonator); + when(ugi.getUserName()).thenReturn("victim"); + when(ugi.getGroups()).thenReturn(Arrays.asList("groupVictim")); + + // Exception should be thrown before applying config + LambdaTestUtils.intercept( + AuthorizationException.class, + "User: impersonator is not allowed to impersonate victim", + () -> ProxyUsers.authorize(ugi, LOOPBACK_ADDRESS)); + + // refresh will look at configuration on the server side + // add additional resource with the new value + // so the server side will pick it up + String tfile = "testRouterRefreshSuperUserGroupsConfiguration_rsrc.xml"; + ArrayList keys = new ArrayList<>( + Arrays.asList( + "hadoop.proxyuser.impersonator.groups", + "hadoop.proxyuser.impersonator.hosts")); + ArrayList values = new ArrayList<>( + Arrays.asList( + "groupVictim", + LOOPBACK_ADDRESS)); + tempResource = addFileBasedConfigResource(tfile, keys, values); + Configuration.addDefaultResource(tfile); + + // Mimic CLI Access + RouterAdmin routerAdmin = new RouterAdmin(conf); + int clientRes = + routerAdmin.run(new String[]{"-refreshSuperUserGroupsConfiguration"}); + + assertEquals("CLI command was not successful", 0, clientRes); + ProxyUsers.authorize(ugi, LOOPBACK_ADDRESS); + } + + private static String addFileBasedConfigResource(String configFileName, + ArrayList keyArray, + ArrayList valueArray) + throws IOException { + + if(keyArray.size() != valueArray.size()) { + throw new IOException("keyArray and valueArray should be equal in size"); + } + + URL url = new Configuration().getResource("hdfs-site.xml"); + Path dir = new Path(URLDecoder.decode(url.getPath(), "UTF-8")).getParent(); + String tmp = dir.toString() + "/" + configFileName; + + StringBuilder configItems = new StringBuilder(); + configItems.append(""); + for (int i = 0; i < keyArray.size(); i++){ + configItems + .append("") + .append("").append(keyArray.get(i)).append("") + .append("").append(valueArray.get(i)).append("") + .append(""); + } + configItems.append(""); + + PrintWriter writer = new PrintWriter(new FileOutputStream(tmp)); + writer.println(configItems.toString()); + writer.close(); + return tmp; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index 92e7c4d0401..7014331a0b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -444,6 +444,7 @@ Usage: [-getDisabledNameservices] [-refresh] [-refreshRouterArgs [arg1..argn]] + [-refreshSuperUserGroupsConfiguration] | COMMAND\_OPTION | Description | |:---- |:---- | @@ -459,6 +460,7 @@ Usage: | `-getDisabledNameservices` | Get the name services that are disabled in the federation. | | `-refresh` | Update mount table cache of the connected router. | | `refreshRouterArgs` \ \ [arg1..argn] | To trigger a runtime-refresh of the resource specified by \ on \. For example, to enable white list checking, we just need to send a refresh command other than restart the router server. | +| `-refreshSuperUserGroupsConfiguration` | Refresh superuser proxy groups mappings on Router. | The commands for managing Router-based federation. See [Mount table management](../hadoop-hdfs-rbf/HDFSRouterFederation.html#Mount_table_management) for more info.