diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt index 63edafd4bcc..f8289ac66cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt @@ -72,6 +72,8 @@ fs-encryption (Unreleased) HDFS-6692. Add more HDFS encryption tests. (wang) + HDFS-6780. Batch the encryption zones listing API. (wang) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index d90c7076e1b..04acb037665 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -150,6 +150,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.EncryptionZone; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneIterator; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -2857,13 +2858,10 @@ public void createEncryptionZone(String src, String keyName) } } - public List listEncryptionZones() throws IOException { + public RemoteIterator listEncryptionZones() + throws IOException { checkOpen(); - try { - return namenode.listEncryptionZones(); - } catch (RemoteException re) { - throw re.unwrapRemoteException(AccessControlException.class); - } + return new EncryptionZoneIterator(namenode); } public void setXAttr(String src, String name, byte[] value, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index c16c15d34b0..d884e5f7c88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -566,7 +566,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_TRUSTEDCHANNEL_RESOLVER_CLASS = "dfs.trustedchannel.resolver.class"; public static final String DFS_DATA_TRANSFER_PROTECTION_KEY = "dfs.data.transfer.protection"; public static final String DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY = "dfs.data.transfer.saslproperties.resolver.class"; - + public static final int DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES_DEFAULT = 100; + public static final String DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES = "dfs.namenode.list.encryption.zones.num.responses"; + // Journal-node related configs. These are read on the JN side. public static final String DFS_JOURNALNODE_EDITS_DIR_KEY = "dfs.journalnode.edits.dir"; public static final String DFS_JOURNALNODE_EDITS_DIR_DEFAULT = "/tmp/hadoop/dfs/journalnode/"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index eccd563b270..52721962531 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -1805,7 +1805,8 @@ public void createEncryptionZone(Path path, String keyName) } /* HDFS only */ - public List listEncryptionZones() throws IOException { + public RemoteIterator listEncryptionZones() + throws IOException { return dfs.listEncryptionZones(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index 0a22d9dd3f4..1a8dce3acbf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -248,16 +248,17 @@ public void createEncryptionZone(Path path, String keyName) } /** - * Return a list of all {@link EncryptionZone}s in the HDFS hierarchy which - * are visible to the caller. If the caller is an HDFS superuser, - * then the key name of each encryption zone will also be provided. - * - * @throws IOException if there was a general IO exception - * - * @return List the list of Encryption Zones that the caller has - * access to. + * Returns a RemoteIterator which can be used to list the encryption zones + * in HDFS. For large numbers of encryption zones, the iterator will fetch + * the list of zones in a number of small batches. + *

+ * Since the list is fetched in batches, it does not represent a + * consistent snapshot of the entire list of encryption zones. + *

+ * This method can only be called by HDFS superusers. */ - public List listEncryptionZones() throws IOException { + public RemoteIterator listEncryptionZones() + throws IOException { return dfs.listEncryptionZones(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index effc10d9fe5..a571ac6e389 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1275,16 +1275,15 @@ public void createEncryptionZone(String src, String keyName) throws IOException; /** - * Return a list of all {@EncryptionZone}s in the HDFS hierarchy which are - * visible to the caller. If the caller is the HDFS admin, then the returned - * EncryptionZone instances will have the key id field filled in. If the - * caller is not the HDFS admin, then the EncryptionZone instances will only - * have the path field filled in and only those zones that are visible to the - * user are returned. + * Used to implement cursor-based batched listing of {@EncryptionZone}s. + * + * @param prevId ID of the last item in the previous batch. If there is no + * previous batch, a negative value can be used. + * @return Batch of encryption zones. */ @Idempotent - public List listEncryptionZones() - throws IOException; + public BatchedEntries listEncryptionZones( + long prevId) throws IOException; /** * Set xattr of a file or directory. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java new file mode 100644 index 00000000000..ff308dabf77 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.protocol; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.RemoteIterator; + +/** + * EncryptionZoneIterator is a remote iterator that iterates over encryption + * zones. It supports retrying in case of namenode failover. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class EncryptionZoneIterator implements RemoteIterator { + + private final EncryptionZoneWithIdIterator iterator; + + public EncryptionZoneIterator(ClientProtocol namenode) { + iterator = new EncryptionZoneWithIdIterator(namenode); + } + + @Override + public boolean hasNext() throws IOException { + return iterator.hasNext(); + } + + @Override + public EncryptionZone next() throws IOException { + EncryptionZoneWithId ezwi = iterator.next(); + return ezwi.toEncryptionZone(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithId.java new file mode 100644 index 00000000000..7ed4884bbd5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithId.java @@ -0,0 +1,64 @@ +package org.apache.hadoop.hdfs.protocol; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Internal class similar to an {@link EncryptionZone} which also holds a + * unique id. Used to implement batched listing of encryption zones. + */ +@InterfaceAudience.Private +public class EncryptionZoneWithId extends EncryptionZone { + + final long id; + + public EncryptionZoneWithId(String path, String keyName, long id) { + super(path, keyName); + this.id = id; + } + + public long getId() { + return id; + } + + EncryptionZone toEncryptionZone() { + return new EncryptionZone(getPath(), getKeyName()); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 29) + .append(super.hashCode()) + .append(id) + .toHashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + + EncryptionZoneWithId that = (EncryptionZoneWithId) o; + + if (id != that.id) { + return false; + } + + return true; + } + + @Override + public String toString() { + return "EncryptionZoneWithId [" + + "id=" + id + + ", " + super.toString() + + ']'; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithIdIterator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithIdIterator.java new file mode 100644 index 00000000000..78c7b620535 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithIdIterator.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.protocol; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.BatchedRemoteIterator; + +/** + * Used on the client-side to iterate over the list of encryption zones + * stored on the namenode. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class EncryptionZoneWithIdIterator + extends BatchedRemoteIterator { + + private final ClientProtocol namenode; + + EncryptionZoneWithIdIterator(ClientProtocol namenode) { + super(Long.valueOf(0)); + this.namenode = namenode; + } + + @Override + public BatchedEntries makeRequest(Long prevId) + throws IOException { + return namenode.listEncryptionZones(prevId); + } + + @Override + public Long elementToPrevKey(EncryptionZoneWithId entry) { + return entry.getId(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 7f7fa185ec8..acb0294eee2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -1317,7 +1318,15 @@ public ListEncryptionZonesResponseProto listEncryptionZones( RpcController controller, ListEncryptionZonesRequestProto req) throws ServiceException { try { - return PBHelper.convertListEZResponse(server.listEncryptionZones()); + BatchedEntries entries = server + .listEncryptionZones(req.getId()); + ListEncryptionZonesResponseProto.Builder builder = + ListEncryptionZonesResponseProto.newBuilder(); + builder.setHasMore(entries.hasMore()); + for (int i=0; i listEncryptionZones() throws IOException { + public BatchedEntries listEncryptionZones(long id) + throws IOException { final ListEncryptionZonesRequestProto req = - ListEncryptionZonesRequestProto.newBuilder().build(); + ListEncryptionZonesRequestProto.newBuilder() + .setId(id) + .build(); try { - return PBHelper.convert(rpcProxy.listEncryptionZones(null, req)); + EncryptionZonesProtos.ListEncryptionZonesResponseProto response = + rpcProxy.listEncryptionZones(null, req); + List elements = + Lists.newArrayListWithCapacity(response.getZonesCount()); + for (EncryptionZoneWithIdProto p : response.getZonesList()) { + elements.add(PBHelper.convert(p)); + } + return new BatchedListEntries(elements, + response.getHasMore()); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index e6de79cbffb..230f5455198 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hdfs.protocolPB; import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos + .EncryptionZoneWithIdProto; import java.io.EOFException; import java.io.IOException; @@ -59,7 +61,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.DatanodeLocalInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; -import org.apache.hadoop.hdfs.protocol.EncryptionZone; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.hdfs.protocol.FsAclPermission; @@ -111,8 +113,6 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.NNHAStatusHeartbeatProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterCommandProto; -import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.EncryptionZoneProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; @@ -2264,45 +2264,6 @@ public static List convertXAttrs(List xAttrSpec) { } return xAttrs; } - - public static List convert(ListEncryptionZonesResponseProto a) { - final List ezs = a.getPathsAndKeysList(); - return convertEZ(ezs); - } - - public static ListEncryptionZonesResponseProto convertListEZResponse( - List ezs) { - final ListEncryptionZonesResponseProto.Builder builder = - ListEncryptionZonesResponseProto.newBuilder(); - builder.addAllPathsAndKeys(convertEZProto(ezs)); - return builder.build(); - } - - public static List convertEZProto( - List ezs) { - final ArrayList ret = - Lists.newArrayListWithCapacity(ezs.size()); - for (EncryptionZone a : ezs) { - final EncryptionZoneProto.Builder builder = - EncryptionZoneProto.newBuilder(); - builder.setPath(a.getPath()); - builder.setKeyName(a.getKeyName()); - ret.add(builder.build()); - } - return ret; - } - - public static List convertEZ( - List ezs) { - final ArrayList ret = - Lists.newArrayListWithCapacity(ezs.size()); - for (EncryptionZoneProto a : ezs) { - final EncryptionZone ez = - new EncryptionZone(a.getPath(), a.getKeyName()); - ret.add(ez); - } - return ret; - } public static List convert(GetXAttrsResponseProto a) { List xAttrs = a.getXAttrsList(); @@ -2334,6 +2295,18 @@ public static ListXAttrsResponseProto convertListXAttrsResponse( return builder.build(); } + public static EncryptionZoneWithIdProto convert(EncryptionZoneWithId zone) { + return EncryptionZoneWithIdProto.newBuilder() + .setId(zone.getId()) + .setKeyName(zone.getKeyName()) + .setPath(zone.getPath()).build(); + } + + public static EncryptionZoneWithId convert(EncryptionZoneWithIdProto proto) { + return new EncryptionZoneWithId(proto.getPath(), proto.getKeyName(), + proto.getId()); + } + public static ShortCircuitShmSlotProto convert(SlotId slotId) { return ShortCircuitShmSlotProto.newBuilder(). setShmId(convert(slotId.getShmId())). diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java index a083ea3a87c..143cc66c3ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java @@ -2,22 +2,25 @@ import java.io.IOException; import java.util.EnumSet; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.XAttrSetFlag; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.XAttrHelper; -import org.apache.hadoop.hdfs.protocol.EncryptionZone; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId; import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants .CRYPTO_XATTR_ENCRYPTION_ZONE; @@ -57,17 +60,26 @@ long getINodeId() { } - private final Map encryptionZones; + private final TreeMap encryptionZones; private final FSDirectory dir; + private final int maxListEncryptionZonesResponses; /** * Construct a new EncryptionZoneManager. * * @param dir Enclosing FSDirectory */ - public EncryptionZoneManager(FSDirectory dir) { + public EncryptionZoneManager(FSDirectory dir, Configuration conf) { this.dir = dir; - encryptionZones = new HashMap(); + encryptionZones = new TreeMap(); + maxListEncryptionZonesResponses = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, + DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES_DEFAULT + ); + Preconditions.checkArgument(maxListEncryptionZonesResponses >= 0, + DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES + " " + + "must be a positive integer." + ); } /** @@ -236,17 +248,30 @@ XAttr createEncryptionZone(String src, String keyName) } /** - * Return the current list of encryption zones. + * Cursor-based listing of encryption zones. *

* Called while holding the FSDirectory lock. */ - List listEncryptionZones() throws IOException { + BatchedListEntries listEncryptionZones(long prevId) + throws IOException { assert dir.hasReadLock(); - final List ret = - Lists.newArrayListWithExpectedSize(encryptionZones.size()); - for (EncryptionZoneInt ezi : encryptionZones.values()) { - ret.add(new EncryptionZone(getFullPathName(ezi), ezi.getKeyName())); + NavigableMap tailMap = encryptionZones.tailMap + (prevId, false); + final int numResponses = Math.min(maxListEncryptionZonesResponses, + tailMap.size()); + final List zones = + Lists.newArrayListWithExpectedSize(numResponses); + + int count = 0; + for (EncryptionZoneInt ezi : tailMap.values()) { + zones.add(new EncryptionZoneWithId(getFullPathName(ezi), + ezi.getKeyName(), ezi.getINodeId())); + count++; + if (count >= numResponses) { + break; + } } - return ret; + final boolean hasMore = (numResponses < tailMap.size()); + return new BatchedListEntries(zones, hasMore); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index f3ef5c3226b..bb687d7940d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO; import static org.apache.hadoop.util.Time.now; @@ -58,7 +59,7 @@ import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DirectoryListing; -import org.apache.hadoop.hdfs.protocol.EncryptionZone; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId; import org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException; import org.apache.hadoop.hdfs.protocol.FSLimitException.PathComponentTooLongException; import org.apache.hadoop.hdfs.protocol.FsAclPermission; @@ -227,7 +228,7 @@ public int getWriteHoldCount() { nameCache = new NameCache(threshold); namesystem = ns; - ezManager = new EncryptionZoneManager(this); + ezManager = new EncryptionZoneManager(this, conf); } private FSNamesystem getFSNamesystem() { @@ -2646,10 +2647,11 @@ XAttr createEncryptionZone(String src, String keyName) } } - List listEncryptionZones() throws IOException { + BatchedListEntries listEncryptionZones(long prevId) + throws IOException { readLock(); try { - return ezManager.listEncryptionZones(); + return ezManager.listEncryptionZones(prevId); } finally { readUnlock(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index e4c7509ea02..da9dcfd1f9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -183,6 +183,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.EncryptionZone; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; @@ -8559,7 +8560,8 @@ private void createEncryptionZoneInt(final String srcArg, String keyName, logAuditEvent(true, "createEncryptionZone", srcArg, null, resultingStat); } - List listEncryptionZones() throws IOException { + BatchedListEntries listEncryptionZones(long prevId) + throws IOException { boolean success = false; checkSuperuserPrivilege(); checkOperation(OperationCategory.READ); @@ -8567,7 +8569,8 @@ List listEncryptionZones() throws IOException { try { checkSuperuserPrivilege(); checkOperation(OperationCategory.READ); - final List ret = dir.listEncryptionZones(); + final BatchedListEntries ret = + dir.listEncryptionZones(prevId); success = true; return ret; } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index ac721dff7f1..9fb0c33b6b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -77,7 +77,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; -import org.apache.hadoop.hdfs.protocol.EncryptionZone; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.FSLimitException; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -1432,8 +1432,9 @@ public void createEncryptionZone(String src, String keyName) } @Override - public List listEncryptionZones() throws IOException { - return namesystem.listEncryptionZones(); + public BatchedEntries listEncryptionZones( + long prevId) throws IOException { + return namesystem.listEncryptionZones(prevId); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java index 28aaef2e455..fcad730075e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.tools.TableListing; @@ -201,8 +202,9 @@ public int run(Configuration conf, List args) throws IOException { final TableListing listing = new TableListing.Builder() .addField("").addField("", true) .wrapWidth(MAX_LINE_WIDTH).hideHeaders().build(); - final List ezs = dfs.listEncryptionZones(); - for (EncryptionZone ez : ezs) { + final RemoteIterator it = dfs.listEncryptionZones(); + while (it.hasNext()) { + EncryptionZone ez = it.next(); listing.addRow(ez.getPath(), ez.getKeyName()); } System.out.println(listing.toString()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto index 391b0aa5ff5..fadaef1b9d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto @@ -42,13 +42,16 @@ message CreateEncryptionZoneResponseProto { } message ListEncryptionZonesRequestProto { + required int64 id = 1; } -message EncryptionZoneProto { +message EncryptionZoneWithIdProto { required string path = 1; required string keyName = 2; + required int64 id = 3; } message ListEncryptionZonesResponseProto { - repeated EncryptionZoneProto pathsAndKeys = 1; + repeated EncryptionZoneWithIdProto zones = 1; + required bool hasMore = 2; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index fea88166c24..961c214c855 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2052,4 +2052,13 @@ + + dfs.namenode.list.encryption.zones.num.responses + false + When listing encryption zones, the maximum number of zones + that will be returned in a batch. Fetching the list incrementally in + batches improves namenode performance. + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 9a3456fb3bb..b48978c97ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -44,6 +44,7 @@ import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.FileSystemTestWrapper; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.apache.hadoop.hdfs.protocol.EncryptionZone; @@ -90,6 +91,9 @@ public void setup() throws Exception { conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, JavaKeyStoreProvider.SCHEME_NAME + "://file" + testRootDir + "/test.jks" ); + // Lower the batch size for testing + conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, + 2); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE); fs = cluster.getFileSystem(); @@ -114,9 +118,13 @@ public void teardown() { } public void assertNumZones(final int numZones) throws IOException { - final List zones = dfsAdmin.listEncryptionZones(); - assertEquals("Unexpected number of encryption zones!", numZones, - zones.size()); + RemoteIterator it = dfsAdmin.listEncryptionZones(); + int count = 0; + while (it.hasNext()) { + count++; + it.next(); + } + assertEquals("Unexpected number of encryption zones!", numZones, count); } /** @@ -126,9 +134,10 @@ public void assertNumZones(final int numZones) throws IOException { * @throws IOException if a matching zone could not be found */ public void assertZonePresent(String keyName, String path) throws IOException { - final List zones = dfsAdmin.listEncryptionZones(); + final RemoteIterator it = dfsAdmin.listEncryptionZones(); boolean match = false; - for (EncryptionZone zone : zones) { + while (it.hasNext()) { + EncryptionZone zone = it.next(); boolean matchKey = (keyName == null); boolean matchPath = (path == null); if (keyName != null && zone.getKeyName().equals(keyName)) { @@ -282,6 +291,16 @@ public Object run() throws Exception { dfsAdmin.createEncryptionZone(deepZone, TEST_KEY); assertNumZones(++numZones); assertZonePresent(null, deepZone.toString()); + + // Create and list some zones to test batching of listEZ + for (int i=1; i<6; i++) { + final Path zonePath = new Path("/listZone" + i); + fsWrapper.mkdir(zonePath, FsPermission.getDirDefault(), false); + dfsAdmin.createEncryptionZone(zonePath, TEST_KEY); + numZones++; + assertNumZones(numZones); + assertZonePresent(null, zonePath.toString()); + } } /** @@ -369,9 +388,8 @@ public void testReadWrite() throws Exception { // Read them back in and compare byte-by-byte verifyFilesEqual(fs, baseFile, encFile1, len); // Roll the key of the encryption zone - List zones = dfsAdmin.listEncryptionZones(); - assertEquals("Expected 1 EZ", 1, zones.size()); - String keyName = zones.get(0).getKeyName(); + assertNumZones(1); + String keyName = dfsAdmin.listEncryptionZones().next().getKeyName(); cluster.getNamesystem().getProvider().rollNewVersion(keyName); // Read them back in and compare byte-by-byte verifyFilesEqual(fs, baseFile, encFile1, len); @@ -457,14 +475,12 @@ public void testCipherSuiteNegotiation() throws Exception { @Test(timeout = 120000) public void testCreateEZWithNoProvider() throws Exception { - + // Unset the key provider and make sure EZ ops don't work final Configuration clusterConf = cluster.getConfiguration(0); clusterConf.set(KeyProviderFactory.KEY_PROVIDER_PATH, ""); cluster.restartNameNode(true); cluster.waitActive(); - /* Test failure of create EZ on a directory that doesn't exist. */ final Path zone1 = new Path("/zone1"); - /* Normal creation of an EZ */ fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true); try { dfsAdmin.createEncryptionZone(zone1, TEST_KEY); @@ -476,8 +492,7 @@ public void testCreateEZWithNoProvider() throws Exception { JavaKeyStoreProvider.SCHEME_NAME + "://file" + testRootDir + "/test.jks" ); // Try listing EZs as well - List zones = dfsAdmin.listEncryptionZones(); - assertEquals("Expected no zones", 0, zones.size()); + assertNumZones(0); } private class MyInjector extends EncryptionFaultInjector {