From dccdedca5b839c013dd887221ac614d53b11b337 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 14 Aug 2013 02:55:46 +0000 Subject: [PATCH 01/76] Create a new branch from trunk for HDFS-2832: Enable support for heterogeneous storages in HDFS. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1513717 13f79535-47bb-0310-9956-ffa450edef68 From 5e27288c1e553eb98f605ef60f7357b39a11eac1 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 21 Aug 2013 23:07:29 +0000 Subject: [PATCH 02/76] HDFS-4985. Add storage type to the protocol and expose it in block report and block locations git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1516323 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/hdfs/StorageType.java | 45 ++++++++++++ .../hadoop/hdfs/protocol/LocatedBlock.java | 18 +++++ .../hadoop/hdfs/protocolPB/PBHelper.java | 70 +++++++++++++++++-- .../hdfs/server/protocol/DatanodeStorage.java | 22 ++++-- .../src/main/proto/DatanodeProtocol.proto | 1 + .../hadoop-hdfs/src/main/proto/hdfs.proto | 8 +++ .../hadoop/hdfs/protocolPB/TestPBHelper.java | 61 +++++++++++++--- 7 files changed, 206 insertions(+), 19 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java new file mode 100644 index 00000000000..3cdd01b959a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Defines the types of supported storage media. The default storage + * medium is assumed to be DISK. + * + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public enum StorageType { + DISK(1), + SSD(2); + public static StorageType DEFAULT = DISK; + + private final int storageType; + + StorageType(int medium) { + storageType = medium; + } + + public int getStorageType() { + return this.storageType; + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java index d9da5b845b7..93724d7569e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.protocol; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.security.token.Token; @@ -34,6 +35,8 @@ public class LocatedBlock { private ExtendedBlock b; private long offset; // offset of the first byte of the block in the file private DatanodeInfo[] locs; + // Storage type for each replica, if reported. + private StorageType[] storageTypes; // corrupt flag is true if all of the replicas of a block are corrupt. // else false. If block has few corrupt replicas, they are filtered and // their locations are not part of this object @@ -50,6 +53,12 @@ public class LocatedBlock { public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, long startOffset, boolean corrupt) { + this(b, locs, null, startOffset, corrupt); + } + + public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, + StorageType[] storageTypes, long startOffset, + boolean corrupt) { this.b = b; this.offset = startOffset; this.corrupt = corrupt; @@ -58,6 +67,7 @@ public class LocatedBlock { } else { this.locs = locs; } + this.storageTypes = storageTypes; } public Token getBlockToken() { @@ -75,6 +85,14 @@ public class LocatedBlock { public DatanodeInfo[] getLocations() { return locs; } + + public void setStorageTypes(StorageType[] storageTypes) { + this.storageTypes = storageTypes; + } + + public StorageType[] getStorageTypes() { + return storageTypes; + } public long getStartOffset() { return offset; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index a3b60a765df..3f015bdb09b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -25,9 +25,11 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.List; +import com.google.common.base.Preconditions; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.proto.HAServiceProtocolProtos; @@ -92,6 +94,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.File import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto.Builder; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlocksProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeCommandProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto.NamenodeRoleProto; @@ -566,6 +569,12 @@ public class PBHelper { for (int i = 0; i < locs.length; i++) { builder.addLocs(i, PBHelper.convert(locs[i])); } + StorageType[] storageTypes = b.getStorageTypes(); + if (storageTypes != null) { + for (int i = 0; i < storageTypes.length; ++i) { + builder.addStorageTypes(PBHelper.convertStorageType(storageTypes[i])); + } + } return builder.setB(PBHelper.convert(b.getBlock())) .setBlockToken(PBHelper.convert(b.getBlockToken())) .setCorrupt(b.isCorrupt()).setOffset(b.getStartOffset()).build(); @@ -578,8 +587,22 @@ public class PBHelper { for (int i = 0; i < locs.size(); i++) { targets[i] = PBHelper.convert(locs.get(i)); } + + List storageTypesList = proto.getStorageTypesList(); + StorageType[] storageTypes = new StorageType[locs.size()]; + + + // The media should correspond to targets 1:1. If not then + // ignore the media information (left as default). + if ((storageTypesList != null) && + (storageTypesList.size() == locs.size())) { + for (int i = 0; i < storageTypesList.size(); ++i) { + storageTypes[i] = PBHelper.convertType(storageTypesList.get(i)); + } + } + LocatedBlock lb = new LocatedBlock(PBHelper.convert(proto.getB()), targets, - proto.getOffset(), proto.getCorrupt()); + storageTypes, proto.getOffset(), proto.getCorrupt()); lb.setBlockToken(PBHelper.convert(proto.getBlockToken())); return lb; } @@ -1327,11 +1350,12 @@ public class PBHelper { public static DatanodeStorageProto convert(DatanodeStorage s) { return DatanodeStorageProto.newBuilder() - .setState(PBHelper.convert(s.getState())) + .setState(PBHelper.convertState(s.getState())) + .setStorageType(PBHelper.convertStorageType(s.getStorageType())) .setStorageID(s.getStorageID()).build(); } - private static StorageState convert(State state) { + private static StorageState convertState(State state) { switch(state) { case READ_ONLY: return StorageState.READ_ONLY; @@ -1341,11 +1365,34 @@ public class PBHelper { } } - public static DatanodeStorage convert(DatanodeStorageProto s) { - return new DatanodeStorage(s.getStorageID(), PBHelper.convert(s.getState())); + private static StorageTypeProto convertStorageType( + StorageType type) { + switch(type) { + case DISK: + return StorageTypeProto.DISK; + case SSD: + return StorageTypeProto.SSD; + default: + Preconditions.checkState( + false, + "Failed to update StorageTypeProto with new StorageType " + + type.toString()); + return StorageTypeProto.DISK; + } } - private static State convert(StorageState state) { + public static DatanodeStorage convert(DatanodeStorageProto s) { + if (s.hasStorageType()) { + return new DatanodeStorage(s.getStorageID(), + PBHelper.convertState(s.getState()), + PBHelper.convertType(s.getStorageType())); + } else { + return new DatanodeStorage(s.getStorageID(), + PBHelper.convertState(s.getState())); + } + } + + private static State convertState(StorageState state) { switch(state) { case READ_ONLY: return DatanodeStorage.State.READ_ONLY; @@ -1355,6 +1402,17 @@ public class PBHelper { } } + private static StorageType convertType(StorageTypeProto type) { + switch(type) { + case DISK: + return StorageType.DISK; + case SSD: + return StorageType.SSD; + default: + return StorageType.DEFAULT; + } + } + public static StorageReportProto convert(StorageReport r) { return StorageReportProto.newBuilder() .setBlockPoolUsed(r.getBlockPoolUsed()).setCapacity(r.getCapacity()) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java index 618708fcdef..d8f1e945f5b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.protocol; +import org.apache.hadoop.hdfs.StorageType; + /** * Class captures information of a storage in Datanode. */ @@ -29,18 +31,26 @@ public class DatanodeStorage { private final String storageID; private final State state; + private final StorageType storageType; /** - * Create a storage with {@link State#NORMAL}. + * Create a storage with {@link State#NORMAL} and + * {@link org.apache.hadoop.hdfs.StorageType#DEFAULT}. + * * @param storageID */ public DatanodeStorage(String storageID) { - this(storageID, State.NORMAL); + this(storageID, State.NORMAL, StorageType.DEFAULT); } public DatanodeStorage(String sid, State s) { - storageID = sid; - state = s; + this(sid, s, StorageType.DEFAULT); + } + + public DatanodeStorage(String sid, State s, StorageType sm) { + this.storageID = sid; + this.state = s; + this.storageType = sm; } public String getStorageID() { @@ -50,4 +60,8 @@ public class DatanodeStorage { public State getState() { return state; } + + public StorageType getStorageType() { + return storageType; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto index bc5461be567..fe9ddce7c36 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto @@ -55,6 +55,7 @@ message DatanodeStorageProto { required string storageID = 1; // Unique identifier for the storage optional StorageState state = 2 [default = NORMAL]; + optional StorageTypeProto storageType = 3; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 085f629a8d3..9709e888480 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -113,6 +113,13 @@ message FsPermissionProto { required uint32 perm = 1; // Actually a short - only 16bits used } +/** + * Types of recognized storage media. + */ +enum StorageTypeProto { + DISK = 1; + SSD = 2; +} /** * A LocatedBlock gives information about a block and its location. @@ -126,6 +133,7 @@ message LocatedBlockProto { // their locations are not part of this object required hadoop.common.TokenProto blockToken = 5; + repeated StorageTypeProto storageTypes = 6; } message DataEncryptionKeyProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index b6c2f6ec29e..05bb5e977a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -17,13 +17,16 @@ */ package org.apache.hadoop.hdfs.protocolPB; +import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; @@ -35,6 +38,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockRecoveryCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeRegistrationProto; +import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeStorageProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockWithLocationsProto; @@ -57,17 +61,9 @@ import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature; -import org.apache.hadoop.hdfs.server.protocol.BlockCommand; -import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand; +import org.apache.hadoop.hdfs.server.protocol.*; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; -import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations; -import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; -import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; -import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; -import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; -import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; -import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.security.token.Token; @@ -155,6 +151,12 @@ public class TestPBHelper { assertEquals(dn.getIpcPort(), dn2.getIpcPort()); } + void compare(DatanodeStorage dns1, DatanodeStorage dns2) { + assertThat(dns2.getStorageID(), is(dns1.getStorageID())); + assertThat(dns2.getState(), is(dns1.getState())); + assertThat(dns2.getStorageType(), is(dns1.getStorageType())); + } + @Test public void testConvertBlock() { Block b = new Block(1, 100, 3); @@ -428,6 +430,29 @@ public class TestPBHelper { DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h3", AdminStates.NORMAL) }; + StorageType[] media = { + StorageType.DISK, + StorageType.SSD, + StorageType.DISK + }; + LocatedBlock lb = new LocatedBlock( + new ExtendedBlock("bp12", 12345, 10, 53), dnInfos, 5, false); + lb.setBlockToken(new Token( + "identifier".getBytes(), "password".getBytes(), new Text("kind"), + new Text("service"))); + lb.setStorageTypes(media); + return lb; + } + + private LocatedBlock createLocatedBlockNoStorageMedia() { + DatanodeInfo[] dnInfos = { + DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h1", + AdminStates.DECOMMISSION_INPROGRESS), + DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h2", + AdminStates.DECOMMISSIONED), + DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h3", + AdminStates.NORMAL) + }; LocatedBlock lb = new LocatedBlock( new ExtendedBlock("bp12", 12345, 10, 53), dnInfos, 5, false); lb.setBlockToken(new Token( @@ -444,6 +469,14 @@ public class TestPBHelper { compare(lb,lb2); } + @Test + public void testConvertLocatedBlockNoStorageMedia() { + LocatedBlock lb = createLocatedBlockNoStorageMedia(); + LocatedBlockProto lbProto = PBHelper.convert(lb); + LocatedBlock lb2 = PBHelper.convert(lbProto); + compare(lb,lb2); + } + @Test public void testConvertLocatedBlockList() { ArrayList lbl = new ArrayList(); @@ -487,6 +520,16 @@ public class TestPBHelper { compare(reg, reg2); assertEquals(reg.getSoftwareVersion(), reg2.getSoftwareVersion()); } + + @Test + public void TestConvertDatanodeStorage() { + DatanodeStorage dns1 = new DatanodeStorage( + "id1", DatanodeStorage.State.NORMAL, StorageType.SSD); + + DatanodeStorageProto proto = PBHelper.convert(dns1); + DatanodeStorage dns2 = PBHelper.convert(proto); + compare(dns1, dns2); + } @Test public void testConvertBlockCommand() { From 6f699e8ea5d8a48c3daaf8dffa2292ce0524cfdb Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 22 Aug 2013 23:21:07 +0000 Subject: [PATCH 03/76] HDFS-5115. Make StorageID a UUID. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1516666 13f79535-47bb-0310-9956-ffa450edef68 --- .../blockmanagement/DatanodeManager.java | 37 ++--------------- .../hadoop/hdfs/server/datanode/DataNode.java | 41 ++----------------- .../hdfs/server/datanode/DataStorage.java | 7 ++-- .../hdfs/server/protocol/DatanodeStorage.java | 12 ++++++ .../server/datanode/SimulatedFSDataset.java | 15 +++---- .../server/datanode/TestBPOfferService.java | 2 +- .../datanode/TestSimulatedFSDataset.java | 2 +- 7 files changed, 29 insertions(+), 87 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 9d5024fb9b1..e9f4fdb636b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -23,13 +23,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.NavigableMap; -import java.util.TreeMap; +import java.util.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -54,15 +48,8 @@ import org.apache.hadoop.hdfs.server.namenode.HostFileManager.EntrySet; import org.apache.hadoop.hdfs.server.namenode.HostFileManager.MutableEntrySet; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; -import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand; -import org.apache.hadoop.hdfs.server.protocol.BlockCommand; -import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand; +import org.apache.hadoop.hdfs.server.protocol.*; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; -import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; -import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; -import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; -import org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException; -import org.apache.hadoop.hdfs.server.protocol.RegisterCommand; import org.apache.hadoop.hdfs.util.CyclicIteration; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.Server; @@ -657,24 +644,6 @@ public class DatanodeManager { } } - /** - * Generate new storage ID. - * - * @return unique storage ID - * - * Note: that collisions are still possible if somebody will try - * to bring in a data storage from a different cluster. - */ - private String newStorageID() { - String newID = null; - while(newID == null) { - newID = "DS" + Integer.toString(DFSUtil.getRandom().nextInt()); - if (datanodeMap.get(newID) != null) - newID = null; - } - return newID; - } - /** * Register the given datanode with the namenode. NB: the given * registration is mutated and given back to the datanode. @@ -779,7 +748,7 @@ public class DatanodeManager { if ("".equals(nodeReg.getStorageID())) { // this data storage has never been registered // it is either empty or was created by pre-storageID version of DFS - nodeReg.setStorageID(newStorageID()); + nodeReg.setStorageID(DatanodeStorage.newStorageID()); if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug( "BLOCK* NameSystem.registerDatanode: " diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index b86a5caebd1..acb9cf5688d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -67,15 +67,7 @@ import java.net.UnknownHostException; import java.nio.channels.ClosedByInterruptException; import java.nio.channels.SocketChannel; import java.security.PrivilegedExceptionAction; -import java.util.AbstractList; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; @@ -138,12 +130,8 @@ import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; import org.apache.hadoop.hdfs.server.datanode.web.resources.DatanodeWebHdfsMethods; import org.apache.hadoop.hdfs.server.namenode.FileChecksumServlets; import org.apache.hadoop.hdfs.server.namenode.StreamFile; +import org.apache.hadoop.hdfs.server.protocol.*; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; -import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; -import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; -import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol; -import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; -import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.Param; import org.apache.hadoop.http.HttpServer; @@ -1041,30 +1029,7 @@ public class DataNode extends Configured public static void setNewStorageID(DatanodeID dnId) { LOG.info("Datanode is " + dnId); - dnId.setStorageID(createNewStorageId(dnId.getXferPort())); - } - - /** - * @return a unique storage ID of form "DS-randInt-ipaddr-port-timestamp" - */ - static String createNewStorageId(int port) { - // It is unlikely that we will create a non-unique storage ID - // for the following reasons: - // a) SecureRandom is a cryptographically strong random number generator - // b) IP addresses will likely differ on different hosts - // c) DataNode xfer ports will differ on the same host - // d) StorageIDs will likely be generated at different times (in ms) - // A conflict requires that all four conditions are violated. - // NB: The format of this string can be changed in the future without - // requiring that old SotrageIDs be updated. - String ip = "unknownIP"; - try { - ip = DNS.getDefaultIP("default"); - } catch (UnknownHostException ignored) { - LOG.warn("Could not find an IP address for the \"default\" inteface."); - } - int rand = DFSUtil.getSecureRandom().nextInt(Integer.MAX_VALUE); - return "DS-" + rand + "-" + ip + "-" + port + "-" + Time.now(); + dnId.setStorageID(DatanodeStorage.newStorageID()); } /** Ensure the authentication method is kerberos */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index 9d31ffa673e..b04f46cedf1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.StorageInfo; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.Daemon; @@ -106,11 +107,11 @@ public class DataStorage extends Storage { } /** Create an ID for this storage. */ - public synchronized void createStorageID(int datanodePort) { + public synchronized void createStorageID() { if (storageID != null && !storageID.isEmpty()) { return; } - storageID = DataNode.createNewStorageId(datanodePort); + storageID = DatanodeStorage.newStorageID(); } /** @@ -194,7 +195,7 @@ public class DataStorage extends Storage { } // make sure we have storage id set - if not - generate new one - createStorageID(datanode.getXferPort()); + createStorageID(); // 3. Update all storages. Some of them might have just been formatted. this.writeAll(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java index d8f1e945f5b..644b294f506 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hdfs.server.protocol; import org.apache.hadoop.hdfs.StorageType; +import java.util.UUID; + /** * Class captures information of a storage in Datanode. */ @@ -64,4 +66,14 @@ public class DatanodeStorage { public StorageType getStorageType() { return storageType; } + + /** + * Generate new storage ID. The format of this string can be changed + * in the future without requiring that old SotrageIDs be updated. + * + * @return unique storage ID + */ + public static String newStorageID() { + return "DS-" + UUID.randomUUID(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 274e5a3a6bd..800a628a4e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -22,11 +22,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; +import java.util.*; import javax.management.NotCompliantMBeanException; import javax.management.ObjectName; @@ -74,7 +70,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { @Override public SimulatedFSDataset newInstance(DataNode datanode, DataStorage storage, Configuration conf) throws IOException { - return new SimulatedFSDataset(datanode, storage, conf); + return new SimulatedFSDataset(storage, conf); } @Override @@ -386,13 +382,12 @@ public class SimulatedFSDataset implements FsDatasetSpi { private final SimulatedStorage storage; private final String storageId; - public SimulatedFSDataset(DataNode datanode, DataStorage storage, - Configuration conf) { + public SimulatedFSDataset(DataStorage storage, Configuration conf) { if (storage != null) { - storage.createStorageID(datanode.getXferPort()); + storage.createStorageID(); this.storageId = storage.getStorageID(); } else { - this.storageId = "unknownStorageId" + new Random().nextInt(); + this.storageId = "unknownStorageId-" + UUID.randomUUID(); } registerMBean(storageId); this.storage = new SimulatedStorage( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java index 42ea48230ee..db7a54b5c97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java @@ -102,7 +102,7 @@ public class TestBPOfferService { .when(mockDn).getMetrics(); // Set up a simulated dataset with our fake BP - mockFSDataset = Mockito.spy(new SimulatedFSDataset(null, null, conf)); + mockFSDataset = Mockito.spy(new SimulatedFSDataset(null, conf)); mockFSDataset.addBlockPool(FAKE_BPID, conf); // Wire the dataset to the DN. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java index e8630091f65..d03e5ea0252 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java @@ -311,7 +311,7 @@ public class TestSimulatedFSDataset { } private SimulatedFSDataset getSimulatedFSDataset() { - SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, null, conf); + SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf); fsdataset.addBlockPool(bpid, conf); return fsdataset; } From 73d14311bc847a29c2b8ec30bbfbaf59cd3cb713 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 26 Aug 2013 03:18:40 +0000 Subject: [PATCH 04/76] HDFS-5000. DataNode configuration should allow specifying storage type git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1517417 13f79535-47bb-0310-9956-ffa450edef68 --- .../hdfs/server/datanode/BPServiceActor.java | 6 +- .../hadoop/hdfs/server/datanode/DataNode.java | 91 ++++++++++--------- .../hdfs/server/datanode/DataStorage.java | 15 +-- .../server/datanode/TestBlockRecovery.java | 29 ++---- .../hdfs/server/datanode/TestDataDirs.java | 77 +++++++++++++--- 5 files changed, 136 insertions(+), 82 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 75f42f959d7..e2a688af85a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -460,9 +460,9 @@ class BPServiceActor implements Runnable { } private String formatThreadName() { - Collection dataDirs = DataNode.getStorageDirs(dn.getConf()); - return "DataNode: [" + - StringUtils.uriToString(dataDirs.toArray(new URI[0])) + "] " + + Collection dataDirs = + DataNode.getStorageLocations(dn.getConf()); + return "DataNode: [" + dataDirs.toString() + "] " + " heartbeating to " + nnAddr; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index acb9cf5688d..cfb323c527e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -272,7 +272,7 @@ public class DataNode extends Configured private JvmPauseMonitor pauseMonitor; private SecureResources secureResources = null; - private AbstractList dataDirs; + private AbstractList dataDirs; private Configuration conf; private final List usersWithLocalPathAccess; @@ -280,21 +280,12 @@ public class DataNode extends Configured ReadaheadPool readaheadPool; private final boolean getHdfsBlockLocationsEnabled; - /** - * Create the DataNode given a configuration and an array of dataDirs. - * 'dataDirs' is where the blocks are stored. - */ - DataNode(final Configuration conf, - final AbstractList dataDirs) throws IOException { - this(conf, dataDirs, null); - } - /** * Create the DataNode given a configuration, an array of dataDirs, * and a namenode proxy */ - DataNode(final Configuration conf, - final AbstractList dataDirs, + DataNode(final Configuration conf, + final AbstractList dataDirs, final SecureResources resources) throws IOException { super(conf); @@ -711,7 +702,7 @@ public class DataNode extends Configured * @throws IOException */ void startDataNode(Configuration conf, - AbstractList dataDirs, + AbstractList dataDirs, // DatanodeProtocol namenode, SecureResources resources ) throws IOException { @@ -861,7 +852,7 @@ public class DataNode extends Configured * If this is the first block pool to register, this also initializes * the datanode-scoped storage. * - * @param nsInfo the handshake response from the NN. + * @param bpos block pool to initialize and register with the NameNode. * @throws IOException if the NN is inconsistent with the local storage. */ void initBlockPool(BPOfferService bpos) throws IOException { @@ -1688,17 +1679,39 @@ public class DataNode extends Configured printUsage(System.err); return null; } - Collection dataDirs = getStorageDirs(conf); + Collection dataLocations = getStorageLocations(conf); UserGroupInformation.setConfiguration(conf); SecurityUtil.login(conf, DFS_DATANODE_KEYTAB_FILE_KEY, DFS_DATANODE_USER_NAME_KEY); - return makeInstance(dataDirs, conf, resources); + return makeInstance(dataLocations, conf, resources); } - static Collection getStorageDirs(Configuration conf) { - Collection dirNames = - conf.getTrimmedStringCollection(DFS_DATANODE_DATA_DIR_KEY); - return Util.stringCollectionAsURIs(dirNames); + static Collection parseStorageLocations( + Collection rawLocations) { + List locations = + new ArrayList(rawLocations.size()); + + for(String locationString : rawLocations) { + StorageLocation location; + try { + location = StorageLocation.parse(locationString); + } catch (IOException ioe) { + LOG.error("Failed to parse storage location " + locationString); + continue; + } catch (IllegalArgumentException iae) { + LOG.error(iae.toString()); + continue; + } + + locations.add(location); + } + + return locations; + } + + static Collection getStorageLocations(Configuration conf) { + return parseStorageLocations( + conf.getTrimmedStringCollection(DFS_DATANODE_DATA_DIR_KEY)); } /** Instantiate & Start a single datanode daemon and wait for it to finish. @@ -1764,51 +1777,45 @@ public class DataNode extends Configured * no directory from this directory list can be created. * @throws IOException */ - static DataNode makeInstance(Collection dataDirs, Configuration conf, - SecureResources resources) throws IOException { + static DataNode makeInstance(Collection dataDirs, + Configuration conf, SecureResources resources) throws IOException { LocalFileSystem localFS = FileSystem.getLocal(conf); FsPermission permission = new FsPermission( conf.get(DFS_DATANODE_DATA_DIR_PERMISSION_KEY, DFS_DATANODE_DATA_DIR_PERMISSION_DEFAULT)); DataNodeDiskChecker dataNodeDiskChecker = new DataNodeDiskChecker(permission); - ArrayList dirs = - getDataDirsFromURIs(dataDirs, localFS, dataNodeDiskChecker); + ArrayList locations = + checkStorageLocations(dataDirs, localFS, dataNodeDiskChecker); DefaultMetricsSystem.initialize("DataNode"); - assert dirs.size() > 0 : "number of data directories should be > 0"; - return new DataNode(conf, dirs, resources); + assert locations.size() > 0 : "number of data directories should be > 0"; + return new DataNode(conf, locations, resources); } // DataNode ctor expects AbstractList instead of List or Collection... - static ArrayList getDataDirsFromURIs(Collection dataDirs, + static ArrayList checkStorageLocations( + Collection dataDirs, LocalFileSystem localFS, DataNodeDiskChecker dataNodeDiskChecker) throws IOException { - ArrayList dirs = new ArrayList(); + ArrayList locations = new ArrayList(); StringBuilder invalidDirs = new StringBuilder(); - for (URI dirURI : dataDirs) { - if (!"file".equalsIgnoreCase(dirURI.getScheme())) { - LOG.warn("Unsupported URI schema in " + dirURI + ". Ignoring ..."); - invalidDirs.append("\"").append(dirURI).append("\" "); - continue; - } - // drop any (illegal) authority in the URI for backwards compatibility - File dir = new File(dirURI.getPath()); + for (StorageLocation location : dataDirs) { try { - dataNodeDiskChecker.checkDir(localFS, new Path(dir.toURI())); - dirs.add(dir); + dataNodeDiskChecker.checkDir(localFS, new Path(location.getUri())); + locations.add(location); } catch (IOException ioe) { LOG.warn("Invalid " + DFS_DATANODE_DATA_DIR_KEY + " " - + dir + " : ", ioe); - invalidDirs.append("\"").append(dir.getCanonicalPath()).append("\" "); + + location.getFile() + " : ", ioe); + invalidDirs.append("\"").append(location.getFile().getCanonicalPath()).append("\" "); } } - if (dirs.size() == 0) { + if (locations.size() == 0) { throw new IOException("All directories in " + DFS_DATANODE_DATA_DIR_KEY + " are invalid: " + invalidDirs); } - return dirs; + return locations; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index b04f46cedf1..7fbe13c90c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -129,7 +129,8 @@ public class DataStorage extends Storage { * @throws IOException */ synchronized void recoverTransitionRead(DataNode datanode, - NamespaceInfo nsInfo, Collection dataDirs, StartupOption startOpt) + NamespaceInfo nsInfo, Collection dataDirs, + StartupOption startOpt) throws IOException { if (initialized) { // DN storage has been initialized, no need to do anything @@ -145,8 +146,8 @@ public class DataStorage extends Storage { // Format and recover. this.storageDirs = new ArrayList(dataDirs.size()); ArrayList dataDirStates = new ArrayList(dataDirs.size()); - for(Iterator it = dataDirs.iterator(); it.hasNext();) { - File dataDir = it.next(); + for(Iterator it = dataDirs.iterator(); it.hasNext();) { + File dataDir = it.next().getFile(); StorageDirectory sd = new StorageDirectory(dataDir); StorageState curState; try { @@ -215,14 +216,14 @@ public class DataStorage extends Storage { * @throws IOException on error */ void recoverTransitionRead(DataNode datanode, String bpID, NamespaceInfo nsInfo, - Collection dataDirs, StartupOption startOpt) throws IOException { + Collection dataDirs, StartupOption startOpt) throws IOException { // First ensure datanode level format/snapshot/rollback is completed recoverTransitionRead(datanode, nsInfo, dataDirs, startOpt); - + // Create list of storage directories for the block pool Collection bpDataDirs = new ArrayList(); - for(Iterator it = dataDirs.iterator(); it.hasNext();) { - File dnRoot = it.next(); + for(Iterator it = dataDirs.iterator(); it.hasNext();) { + File dnRoot = it.next().getFile(); File bpRoot = BlockPoolSliceStorage.getBpRoot(bpID, new File(dnRoot, STORAGE_DIR_CURRENT)); bpDataDirs.add(bpRoot); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java index a5792ad217f..eb9f1769029 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java @@ -35,6 +35,8 @@ import static org.mockito.Mockito.when; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -44,18 +46,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.*; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.*; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -80,10 +73,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.DataChecksum; import org.apache.log4j.Level; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -121,7 +111,7 @@ public class TestBlockRecovery { * @throws IOException */ @Before - public void startUp() throws IOException { + public void startUp() throws IOException, URISyntaxException { tearDownDone = false; conf = new HdfsConfiguration(); conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, DATA_DIR); @@ -131,11 +121,12 @@ public class TestBlockRecovery { conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); FileSystem.setDefaultUri(conf, "hdfs://" + NN_ADDR.getHostName() + ":" + NN_ADDR.getPort()); - ArrayList dirs = new ArrayList(); + ArrayList locations = new ArrayList(); File dataDir = new File(DATA_DIR); FileUtil.fullyDelete(dataDir); dataDir.mkdirs(); - dirs.add(dataDir); + StorageLocation location = new StorageLocation(new URI(dataDir.getPath())); + locations.add(location); final DatanodeProtocolClientSideTranslatorPB namenode = mock(DatanodeProtocolClientSideTranslatorPB.class); @@ -161,7 +152,7 @@ public class TestBlockRecovery { new DatanodeCommand[0], new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1))); - dn = new DataNode(conf, dirs, null) { + dn = new DataNode(conf, locations, null) { @Override DatanodeProtocolClientSideTranslatorPB connectToNN( InetSocketAddress nnAddr) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java index fc7a3ff01d1..57ee14464cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java @@ -20,11 +20,14 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.*; import java.net.URI; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; +import java.util.*; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.StorageType; import org.junit.Test; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; +import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; import static org.mockito.Mockito.*; @@ -34,19 +37,71 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode.DataNodeDiskChecker; public class TestDataDirs { - @Test (timeout = 10000) - public void testGetDataDirsFromURIs() throws Throwable { + @Test (timeout = 30000) + public void testDataDirParsing() throws Throwable { + Configuration conf = new Configuration(); + ArrayList locations; + File dir0 = new File("/dir0"); + File dir1 = new File("/dir1"); + File dir2 = new File("/dir2"); + File dir3 = new File("/dir3"); + + // Verify that a valid string is correctly parsed, and that storage + // type is not case-sensitive + String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3"; + conf.set(DFS_DATANODE_DATA_DIR_KEY, locations1); + locations = new ArrayList(DataNode.getStorageLocations(conf)); + assertThat(locations.size(), is(4)); + assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(0).getUri(), is(dir0.toURI())); + assertThat(locations.get(1).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(1).getUri(), is(dir1.toURI())); + assertThat(locations.get(2).getStorageType(), is(StorageType.SSD)); + assertThat(locations.get(2).getUri(), is(dir2.toURI())); + assertThat(locations.get(3).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(3).getUri(), is(dir3.toURI())); + + // Verify that an unrecognized storage type is ignored. + String locations2 = "[BadMediaType]/dir0,[ssd]/dir1,[disk]/dir2"; + conf.set(DFS_DATANODE_DATA_DIR_KEY, locations2); + locations = new ArrayList(DataNode.getStorageLocations(conf)); + assertThat(locations.size(), is(3)); + assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(0).getUri(), is(dir0.toURI())); + assertThat(locations.get(1).getStorageType(), is(StorageType.SSD)); + assertThat(locations.get(1).getUri(), is(dir1.toURI())); + assertThat(locations.get(2).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(2).getUri(), is(dir2.toURI())); + + // Assert that a string with no storage type specified is + // correctly parsed and the default storage type is picked up. + String locations3 = "/dir0,/dir1"; + conf.set(DFS_DATANODE_DATA_DIR_KEY, locations3); + locations = new ArrayList(DataNode.getStorageLocations(conf)); + assertThat(locations.size(), is(2)); + assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(0).getUri(), is(dir0.toURI())); + assertThat(locations.get(1).getStorageType(), is(StorageType.DISK)); + assertThat(locations.get(1).getUri(), is(dir1.toURI())); + } + + @Test (timeout = 30000) + public void testDataDirValidation() throws Throwable { DataNodeDiskChecker diskChecker = mock(DataNodeDiskChecker.class); doThrow(new IOException()).doThrow(new IOException()).doNothing() .when(diskChecker).checkDir(any(LocalFileSystem.class), any(Path.class)); LocalFileSystem fs = mock(LocalFileSystem.class); - Collection uris = Arrays.asList(new URI("file:/p1/"), - new URI("file:/p2/"), new URI("file:/p3/")); + AbstractList locations = new ArrayList(); - List dirs = DataNode.getDataDirsFromURIs(uris, fs, diskChecker); - assertEquals("number of valid data dirs", 1, dirs.size()); - String validDir = dirs.iterator().next().getPath(); - assertEquals("p3 should be valid", new File("/p3").getPath(), validDir); + locations.add(new StorageLocation(new URI("file:/p1/"))); + locations.add(new StorageLocation(new URI("file:/p2/"))); + locations.add(new StorageLocation(new URI("file:/p3/"))); + + ArrayList checkedLocations = + DataNode.checkStorageLocations(locations, fs, diskChecker); + assertEquals("number of valid data dirs", 1, checkedLocations.size()); + String validDir = checkedLocations.iterator().next().getFile().getPath(); + assertThat("p3 should be valid", new File("/p3/").getPath(), is(validDir)); } } From 395f0ae56782777024590796a6ecd7c0f6aa5180 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 26 Aug 2013 11:32:26 +0000 Subject: [PATCH 05/76] HDFS-5000. DataNode configuration should allow specifying storage type (Adding file missed in last commit git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1517499 13f79535-47bb-0310-9956-ffa450edef68 --- .../hdfs/server/datanode/StorageLocation.java | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java new file mode 100644 index 00000000000..bc1e57a0038 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.datanode; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.StorageType; +import org.apache.hadoop.hdfs.server.common.Util; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.regex.Matcher; + +import static java.util.regex.Pattern.compile; + +/** + * Encapsulates the URI and storage medium that together describe a + * storage directory. + * The default storage medium is assumed to be DISK, if none is specified. + * + */ +@InterfaceAudience.Private +public class StorageLocation { + public static final Log LOG = LogFactory.getLog(StorageLocation.class); + final StorageType storageType; + final File file; + + // Regular expression that describes a storage uri with a storage type. + // e.g. [Disk]/storages/storage1/ + private static final String rawStringRegex = "^\\[(\\w*)\\](.+)$"; + + StorageLocation(URI uri) { + this(StorageType.DISK, uri); + } + + StorageLocation(StorageType storageType, URI uri) { + this.storageType = storageType; + + if (uri.getScheme() == null || + "file".equalsIgnoreCase(uri.getScheme())) { + // drop any (illegal) authority in the URI for backwards compatibility + this.file = new File(uri.getPath()); + } else { + throw new IllegalArgumentException( + "Got an Unsupported URI schema in " + uri + ". Ignoring ..."); + } + } + + public StorageType getStorageType() { + return this.storageType; + } + + public URI getUri() { + return file.toURI(); + } + + public File getFile() { + return this.file; + } + + /** + * Attempt to parse a storage uri with storage class and URI. The storage + * class component of the uri is case-insensitive. + * + * @param rawLocation Location string of the format [type]uri, where [type] is + * optional. + * @return A StorageLocation object if successfully parsed, null otherwise. + * Does not throw any exceptions. + */ + public static StorageLocation parse(String rawLocation) throws IOException { + Matcher matcher = compile(rawStringRegex).matcher(rawLocation); + StorageType storageType = StorageType.DISK; + String location = rawLocation; + + if (matcher.matches()) { + String classString = matcher.group(1); + location = matcher.group(2); + if (!classString.isEmpty()) { + try { + storageType = StorageType.valueOf(classString.toUpperCase()); + } catch (RuntimeException re) { + LOG.error("Unable to parse storage type: " + re.toString() + + ". Using the default storage type for directory " + + location); + } + } + } + + return new StorageLocation(storageType, Util.stringAsURI(location)); + } + + public String toString() { + return "[" + storageType.toString() + "]" + file.toURI().toString(); + } +} From 5d9d702607913685eab0d8ad077040ddc82bf085 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 28 Aug 2013 06:30:15 +0000 Subject: [PATCH 06/76] HDFS-4987. Namenode changes to track multiple storages per datanode. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1518087 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 4 + .../server/blockmanagement/BlockInfo.java | 132 +++++++++------ .../server/blockmanagement/BlockManager.java | 130 +++++++-------- .../blockmanagement/DatanodeDescriptor.java | 152 ++++++++++++------ .../blockmanagement/DatanodeStorageInfo.java | 142 ++++++++++++++++ .../PendingDataNodeMessages.java | 12 +- .../hdfs/server/namenode/FSNamesystem.java | 33 ++-- .../server/namenode/NameNodeRpcServer.java | 26 +-- .../hadoop/hdfs/TestFileCorruption.java | 2 +- .../server/blockmanagement/TestBlockInfo.java | 124 -------------- .../blockmanagement/TestBlockManager.java | 20 +-- .../TestDatanodeDescriptor.java | 7 +- .../TestPendingDataNodeMessages.java | 4 +- .../TestPendingReplication.java | 4 +- .../TestReplicationPolicy.java | 2 +- .../namenode/metrics/TestNameNodeMetrics.java | 4 +- 16 files changed, 458 insertions(+), 340 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt new file mode 100644 index 00000000000..7ab7c4c1650 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -0,0 +1,4 @@ + BREAKDOWN OF HDFS-2832 ENABLE SUPPORT FOR HETEROGENEOUS STORAGES IN HDFS + + HDFS-4987. Namenode changes to track multiple storages per datanode. + (szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index 8307180cedd..7aa7721a022 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -21,6 +21,7 @@ import java.util.LinkedList; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.util.LightWeightGSet; @@ -39,11 +40,11 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { private LightWeightGSet.LinkedElement nextLinkedElement; /** - * This array contains triplets of references. For each i-th datanode the - * block belongs to triplets[3*i] is the reference to the DatanodeDescriptor - * and triplets[3*i+1] and triplets[3*i+2] are references to the previous and - * the next blocks, respectively, in the list of blocks belonging to this - * data-node. + * This array contains triplets of references. For each i-th storage, the + * block belongs to triplets[3*i] is the reference to the + * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are + * references to the previous and the next blocks, respectively, in the list + * of blocks belonging to this storage. * * Using previous and next in Object triplets is done instead of a * {@link LinkedList} list to efficiently use memory. With LinkedList the cost @@ -86,9 +87,15 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { } DatanodeDescriptor getDatanode(int index) { + DatanodeStorageInfo storage = getStorageInfo(index); + return storage == null ? null : storage.getDatanodeDescriptor(); + } + + DatanodeStorageInfo getStorageInfo(int index) { assert this.triplets != null : "BlockInfo is not initialized"; assert index >= 0 && index*3 < triplets.length : "Index is out of bound"; - return (DatanodeDescriptor)triplets[index*3]; + DatanodeStorageInfo storage = (DatanodeStorageInfo)triplets[index*3]; + return storage; } private BlockInfo getPrevious(int index) { @@ -111,14 +118,10 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { return info; } - private void setDatanode(int index, DatanodeDescriptor node, BlockInfo previous, - BlockInfo next) { + void setStorageInfo(int index, DatanodeStorageInfo storage) { assert this.triplets != null : "BlockInfo is not initialized"; - int i = index * 3; - assert index >= 0 && i+2 < triplets.length : "Index is out of bound"; - triplets[i] = node; - triplets[i+1] = previous; - triplets[i+2] = next; + assert index >= 0 && index*3 < triplets.length : "Index is out of bound"; + triplets[index*3] = storage; } /** @@ -190,22 +193,24 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { } /** - * Add data-node this block belongs to. + * Add a {@link DatanodeStorageInfo} location for a block */ - public boolean addNode(DatanodeDescriptor node) { - if(findDatanode(node) >= 0) // the node is already there + boolean addStorage(DatanodeStorageInfo storage) { + if(findStorageInfo(storage) >= 0) // the node is already there return false; // find the last null node int lastNode = ensureCapacity(1); - setDatanode(lastNode, node, null, null); + setStorageInfo(lastNode, storage); + setNext(lastNode, null); + setPrevious(lastNode, null); return true; } /** - * Remove data-node from the block. + * Remove {@link DatanodeStorageInfo} location for a block */ - public boolean removeNode(DatanodeDescriptor node) { - int dnIndex = findDatanode(node); + boolean removeStorage(DatanodeStorageInfo storage) { + int dnIndex = findStorageInfo(storage); if(dnIndex < 0) // the node is not found return false; assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : @@ -213,10 +218,13 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { // find the last not null node int lastNode = numNodes()-1; // replace current node triplet by the lastNode one - setDatanode(dnIndex, getDatanode(lastNode), getPrevious(lastNode), - getNext(lastNode)); + setStorageInfo(dnIndex, getStorageInfo(lastNode)); + setNext(dnIndex, getNext(lastNode)); + setPrevious(dnIndex, getPrevious(lastNode)); // set the last triplet to null - setDatanode(lastNode, null, null, null); + setStorageInfo(lastNode, null); + setNext(lastNode, null); + setPrevious(lastNode, null); return true; } @@ -236,37 +244,70 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { } return -1; } + /** + * Find specified DatanodeStorageInfo. + * @param dn + * @return index or -1 if not found. + */ + int findStorageInfo(DatanodeInfo dn) { + int len = getCapacity(); + for(int idx = 0; idx < len; idx++) { + DatanodeStorageInfo cur = getStorageInfo(idx); + if(cur == null) + break; + if(cur.getDatanodeDescriptor() == dn) + return idx; + } + return -1; + } + + /** + * Find specified DatanodeStorageInfo. + * @param storageInfo + * @return index or -1 if not found. + */ + int findStorageInfo(DatanodeStorageInfo storageInfo) { + int len = getCapacity(); + for(int idx = 0; idx < len; idx++) { + DatanodeStorageInfo cur = getStorageInfo(idx); + if(cur == storageInfo) + return idx; + if(cur == null) + break; + } + return -1; + } /** * Insert this block into the head of the list of blocks - * related to the specified DatanodeDescriptor. + * related to the specified DatanodeStorageInfo. * If the head is null then form a new list. * @return current block as the new head of the list. */ - public BlockInfo listInsert(BlockInfo head, DatanodeDescriptor dn) { - int dnIndex = this.findDatanode(dn); + BlockInfo listInsert(BlockInfo head, DatanodeStorageInfo storage) { + int dnIndex = this.findStorageInfo(storage); assert dnIndex >= 0 : "Data node is not found: current"; assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : "Block is already in the list and cannot be inserted."; this.setPrevious(dnIndex, null); this.setNext(dnIndex, head); if(head != null) - head.setPrevious(head.findDatanode(dn), this); + head.setPrevious(head.findStorageInfo(storage), this); return this; } /** * Remove this block from the list of blocks - * related to the specified DatanodeDescriptor. + * related to the specified DatanodeStorageInfo. * If this block is the head of the list then return the next block as * the new head. * @return the new head of the list or null if the list becomes - * empty after deletion. + * empy after deletion. */ - public BlockInfo listRemove(BlockInfo head, DatanodeDescriptor dn) { + BlockInfo listRemove(BlockInfo head, DatanodeStorageInfo storage) { if(head == null) return null; - int dnIndex = this.findDatanode(dn); + int dnIndex = this.findStorageInfo(storage); if(dnIndex < 0) // this block is not on the data-node list return head; @@ -275,33 +316,20 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { this.setNext(dnIndex, null); this.setPrevious(dnIndex, null); if(prev != null) - prev.setNext(prev.findDatanode(dn), next); + prev.setNext(prev.findStorageInfo(storage), next); if(next != null) - next.setPrevious(next.findDatanode(dn), prev); + next.setPrevious(next.findStorageInfo(storage), prev); if(this == head) // removing the head head = next; return head; } - /** - * Remove this block from the list of blocks related to the specified - * DatanodeDescriptor. Insert it into the head of the list of blocks. - * - * @return the new head of the list. - */ - public BlockInfo moveBlockToHead(BlockInfo head, DatanodeDescriptor dn, - int curIndex, int headIndex) { - if (head == this) { - return this; - } - BlockInfo next = this.setNext(curIndex, head); - BlockInfo prev = this.setPrevious(curIndex, null); - - head.setPrevious(headIndex, this); - prev.setNext(prev.findDatanode(dn), next); - if (next != null) - next.setPrevious(next.findDatanode(dn), prev); - return this; + int listCount(DatanodeStorageInfo storage) { + int count = 0; + for(BlockInfo cur = this; cur != null; + cur = cur.getNext(cur.findStorageInfo(storage))) + count++; + return count; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 512a052ddb5..77d75a4eb57 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.Queue; import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; @@ -68,8 +69,10 @@ import org.apache.hadoop.hdfs.server.protocol.BlockCommand; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; +import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.util.LightWeightLinkedSet; import org.apache.hadoop.net.Node; import org.apache.hadoop.security.UserGroupInformation; @@ -1034,7 +1037,7 @@ public class BlockManager { * for logging purposes */ public void findAndMarkBlockAsCorrupt(final ExtendedBlock blk, - final DatanodeInfo dn, String reason) throws IOException { + final DatanodeInfo dn, String storageID, String reason) throws IOException { assert namesystem.hasWriteLock(); final BlockInfo storedBlock = getStoredBlock(blk.getLocalBlock()); if (storedBlock == null) { @@ -1046,11 +1049,11 @@ public class BlockManager { + blk + " not found"); return; } - markBlockAsCorrupt(new BlockToMarkCorrupt(storedBlock, reason), dn); + markBlockAsCorrupt(new BlockToMarkCorrupt(storedBlock, reason), dn, storageID); } private void markBlockAsCorrupt(BlockToMarkCorrupt b, - DatanodeInfo dn) throws IOException { + DatanodeInfo dn, String storageID) throws IOException { DatanodeDescriptor node = getDatanodeManager().getDatanode(dn); if (node == null) { throw new IOException("Cannot mark " + b @@ -1066,7 +1069,7 @@ public class BlockManager { } // Add replica to the data-node if it is not already there - node.addBlock(b.stored); + node.addBlock(storageID, b.stored); // Add this replica to corruptReplicas Map corruptReplicas.addToCorruptReplicasMap(b.corrupted, node, b.reason); @@ -1601,10 +1604,11 @@ public class BlockManager { } /** - * The given datanode is reporting all its blocks. - * Update the (machine-->blocklist) and (block-->machinelist) maps. + * The given storage is reporting all its blocks. + * Update the (storage-->block list) and (block-->storage list) maps. */ - public void processReport(final DatanodeID nodeID, final String poolId, + public void processReport(final DatanodeID nodeID, + final DatanodeStorage storage, final String poolId, final BlockListAsLongs newReport) throws IOException { namesystem.writeLock(); final long startTime = Time.now(); //after acquiring write lock @@ -1628,9 +1632,9 @@ public class BlockManager { if (node.numBlocks() == 0) { // The first block report can be processed a lot more efficiently than // ordinary block reports. This shortens restart times. - processFirstBlockReport(node, newReport); + processFirstBlockReport(node, storage.getStorageID(), newReport); } else { - processReport(node, newReport); + processReport(node, storage, newReport); } // Now that we have an up-to-date block report, we know that any @@ -1691,28 +1695,31 @@ public class BlockManager { } private void processReport(final DatanodeDescriptor node, + final DatanodeStorage storage, final BlockListAsLongs report) throws IOException { // Normal case: // Modify the (block-->datanode) map, according to the difference // between the old and new block report. // Collection toAdd = new LinkedList(); - Collection toRemove = new LinkedList(); + Collection toRemove = new TreeSet(); Collection toInvalidate = new LinkedList(); Collection toCorrupt = new LinkedList(); Collection toUC = new LinkedList(); - reportDiff(node, report, toAdd, toRemove, toInvalidate, toCorrupt, toUC); + reportDiff(node, storage, report, + toAdd, toRemove, toInvalidate, toCorrupt, toUC); // Process the blocks on each queue for (StatefulBlockInfo b : toUC) { - addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState); + addStoredBlockUnderConstruction(b.storedBlock, node, + storage.getStorageID(), b.reportedState); } for (Block b : toRemove) { removeStoredBlock(b, node); } int numBlocksLogged = 0; for (BlockInfo b : toAdd) { - addStoredBlock(b, node, null, numBlocksLogged < maxNumBlocksToLog); + addStoredBlock(b, node, storage.getStorageID(), null, numBlocksLogged < maxNumBlocksToLog); numBlocksLogged++; } if (numBlocksLogged > maxNumBlocksToLog) { @@ -1726,7 +1733,7 @@ public class BlockManager { addToInvalidates(b, node); } for (BlockToMarkCorrupt b : toCorrupt) { - markBlockAsCorrupt(b, node); + markBlockAsCorrupt(b, node, storage.getStorageID()); } } @@ -1742,6 +1749,7 @@ public class BlockManager { * @throws IOException */ private void processFirstBlockReport(final DatanodeDescriptor node, + final String storageID, final BlockListAsLongs report) throws IOException { if (report == null) return; assert (namesystem.hasWriteLock()); @@ -1754,7 +1762,7 @@ public class BlockManager { if (shouldPostponeBlocksFromFuture && namesystem.isGenStampInFuture(iblk)) { - queueReportedBlock(node, iblk, reportedState, + queueReportedBlock(node, storageID, iblk, reportedState, QUEUE_REASON_FUTURE_GENSTAMP); continue; } @@ -1771,10 +1779,10 @@ public class BlockManager { if (shouldPostponeBlocksFromFuture) { // In the Standby, we may receive a block report for a file that we // just have an out-of-date gen-stamp or state for, for example. - queueReportedBlock(node, iblk, reportedState, + queueReportedBlock(node, storageID, iblk, reportedState, QUEUE_REASON_CORRUPT_STATE); } else { - markBlockAsCorrupt(c, node); + markBlockAsCorrupt(c, node, storageID); } continue; } @@ -1787,25 +1795,26 @@ public class BlockManager { } //add replica if appropriate if (reportedState == ReplicaState.FINALIZED) { - addStoredBlockImmediate(storedBlock, node); + addStoredBlockImmediate(storedBlock, node, storageID); } } } - private void reportDiff(DatanodeDescriptor dn, + private void reportDiff(DatanodeDescriptor dn, DatanodeStorage storage, BlockListAsLongs newReport, Collection toAdd, // add to DatanodeDescriptor Collection toRemove, // remove from DatanodeDescriptor Collection toInvalidate, // should be removed from DN Collection toCorrupt, // add to corrupt replicas list Collection toUC) { // add to under-construction list - // place a delimiter in the list which separates blocks - // that have been reported from those that have not - BlockInfo delimiter = new BlockInfo(new Block(), 1); - boolean added = dn.addBlock(delimiter); - assert added : "Delimiting block cannot be present in the node"; - int headIndex = 0; //currently the delimiter is in the head of the list - int curIndex; + + dn.updateStorage(storage); + + // add all blocks to remove list + for(Iterator it = dn.getBlockIterator(storage.getStorageID()); + it.hasNext(); ) { + toRemove.add(it.next()); + } if (newReport == null) newReport = new BlockListAsLongs(); @@ -1814,20 +1823,10 @@ public class BlockManager { while(itBR.hasNext()) { Block iblk = itBR.next(); ReplicaState iState = itBR.getCurrentReplicaState(); - BlockInfo storedBlock = processReportedBlock(dn, iblk, iState, - toAdd, toInvalidate, toCorrupt, toUC); - // move block to the head of the list - if (storedBlock != null && (curIndex = storedBlock.findDatanode(dn)) >= 0) { - headIndex = dn.moveBlockToHead(storedBlock, curIndex, headIndex); - } + BlockInfo storedBlock = processReportedBlock(dn, storage.getStorageID(), + iblk, iState, toAdd, toInvalidate, toCorrupt, toUC); + toRemove.remove(storedBlock); } - // collect blocks that have not been reported - // all of them are next to the delimiter - Iterator it = new DatanodeDescriptor.BlockIterator( - delimiter.getNext(0), dn); - while(it.hasNext()) - toRemove.add(it.next()); - dn.removeBlock(delimiter); } /** @@ -1861,7 +1860,8 @@ public class BlockManager { * @return the up-to-date stored block, if it should be kept. * Otherwise, null. */ - private BlockInfo processReportedBlock(final DatanodeDescriptor dn, + private BlockInfo processReportedBlock(final DatanodeDescriptor dn, + final String storageID, final Block block, final ReplicaState reportedState, final Collection toAdd, final Collection toInvalidate, @@ -1876,7 +1876,7 @@ public class BlockManager { if (shouldPostponeBlocksFromFuture && namesystem.isGenStampInFuture(block)) { - queueReportedBlock(dn, block, reportedState, + queueReportedBlock(dn, storageID, block, reportedState, QUEUE_REASON_FUTURE_GENSTAMP); return null; } @@ -1911,7 +1911,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block // If the block is an out-of-date generation stamp or state, // but we're the standby, we shouldn't treat it as corrupt, // but instead just queue it for later processing. - queueReportedBlock(dn, storedBlock, reportedState, + queueReportedBlock(dn, storageID, storedBlock, reportedState, QUEUE_REASON_CORRUPT_STATE); } else { toCorrupt.add(c); @@ -1938,7 +1938,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block * standby node. @see PendingDataNodeMessages. * @param reason a textual reason to report in the debug logs */ - private void queueReportedBlock(DatanodeDescriptor dn, Block block, + private void queueReportedBlock(DatanodeDescriptor dn, String storageID, Block block, ReplicaState reportedState, String reason) { assert shouldPostponeBlocksFromFuture; @@ -1948,7 +1948,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block " from datanode " + dn + " for later processing " + "because " + reason + "."); } - pendingDNMessages.enqueueReportedBlock(dn, block, reportedState); + pendingDNMessages.enqueueReportedBlock(dn, storageID, block, reportedState); } /** @@ -1971,8 +1971,8 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block if (LOG.isDebugEnabled()) { LOG.debug("Processing previouly queued message " + rbi); } - processAndHandleReportedBlock( - rbi.getNode(), rbi.getBlock(), rbi.getReportedState(), null); + processAndHandleReportedBlock(rbi.getNode(), rbi.getStorageID(), + rbi.getBlock(), rbi.getReportedState(), null); } } @@ -2090,18 +2090,18 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block void addStoredBlockUnderConstruction( BlockInfoUnderConstruction block, - DatanodeDescriptor node, + DatanodeDescriptor node, String storageID, ReplicaState reportedState) throws IOException { block.addReplicaIfNotPresent(node, block, reportedState); if (reportedState == ReplicaState.FINALIZED && block.findDatanode(node) < 0) { - addStoredBlock(block, node, null, true); + addStoredBlock(block, node, storageID, null, true); } } /** * Faster version of - * {@link #addStoredBlock(BlockInfo, DatanodeDescriptor, DatanodeDescriptor, boolean)} + * {@link #addStoredBlock(BlockInfo, DatanodeDescriptor, String, DatanodeDescriptor, boolean)} * , intended for use with initial block report at startup. If not in startup * safe mode, will call standard addStoredBlock(). Assumes this method is * called "immediately" so there is no need to refresh the storedBlock from @@ -2112,17 +2112,17 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block * @throws IOException */ private void addStoredBlockImmediate(BlockInfo storedBlock, - DatanodeDescriptor node) + DatanodeDescriptor node, String storageID) throws IOException { assert (storedBlock != null && namesystem.hasWriteLock()); if (!namesystem.isInStartupSafeMode() || namesystem.isPopulatingReplQueues()) { - addStoredBlock(storedBlock, node, null, false); + addStoredBlock(storedBlock, node, storageID, null, false); return; } // just add it - node.addBlock(storedBlock); + node.addBlock(storageID, storedBlock); // Now check for completion of blocks and safe block count int numCurrentReplica = countLiveNodes(storedBlock); @@ -2145,6 +2145,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block */ private Block addStoredBlock(final BlockInfo block, DatanodeDescriptor node, + String storageID, DatanodeDescriptor delNodeHint, boolean logEveryBlock) throws IOException { @@ -2170,7 +2171,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block assert bc != null : "Block must belong to a file"; // add block to the datanode - boolean added = node.addBlock(storedBlock); + boolean added = node.addBlock(storageID, storedBlock); int curReplicaDelta; if (added) { @@ -2614,7 +2615,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block * The given node is reporting that it received a certain block. */ @VisibleForTesting - void addBlock(DatanodeDescriptor node, Block block, String delHint) + void addBlock(DatanodeDescriptor node, String storageID, Block block, String delHint) throws IOException { // decrement number of blocks scheduled to this datanode. // for a retry request (of DatanodeProtocol#blockReceivedAndDeleted with @@ -2635,11 +2636,12 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block // Modify the blocks->datanode map and node's map. // pendingReplications.decrement(block, node); - processAndHandleReportedBlock(node, block, ReplicaState.FINALIZED, + processAndHandleReportedBlock(node, storageID, block, ReplicaState.FINALIZED, delHintNode); } - private void processAndHandleReportedBlock(DatanodeDescriptor node, Block block, + private void processAndHandleReportedBlock(DatanodeDescriptor node, + String storageID, Block block, ReplicaState reportedState, DatanodeDescriptor delHintNode) throws IOException { // blockReceived reports a finalized block @@ -2647,7 +2649,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block Collection toInvalidate = new LinkedList(); Collection toCorrupt = new LinkedList(); Collection toUC = new LinkedList(); - processReportedBlock(node, block, reportedState, + processReportedBlock(node, storageID, block, reportedState, toAdd, toInvalidate, toCorrupt, toUC); // the block is only in one of the to-do lists // if it is in none then data-node already has it @@ -2655,11 +2657,11 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block : "The block should be only in one of the lists."; for (StatefulBlockInfo b : toUC) { - addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState); + addStoredBlockUnderConstruction(b.storedBlock, node, storageID, b.reportedState); } long numBlocksLogged = 0; for (BlockInfo b : toAdd) { - addStoredBlock(b, node, delHintNode, numBlocksLogged < maxNumBlocksToLog); + addStoredBlock(b, node, storageID, delHintNode, numBlocksLogged < maxNumBlocksToLog); numBlocksLogged++; } if (numBlocksLogged > maxNumBlocksToLog) { @@ -2673,7 +2675,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block addToInvalidates(b, node); } for (BlockToMarkCorrupt b : toCorrupt) { - markBlockAsCorrupt(b, node); + markBlockAsCorrupt(b, node, storageID); } } @@ -2685,7 +2687,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block * This method must be called with FSNamesystem lock held. */ public void processIncrementalBlockReport(final DatanodeID nodeID, - final String poolId, final ReceivedDeletedBlockInfo blockInfos[]) + final String poolId, final StorageReceivedDeletedBlocks srdb) throws IOException { assert namesystem.hasWriteLock(); int received = 0; @@ -2701,19 +2703,19 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block "Got incremental block report from unregistered or dead node"); } - for (ReceivedDeletedBlockInfo rdbi : blockInfos) { + for (ReceivedDeletedBlockInfo rdbi : srdb.getBlocks()) { switch (rdbi.getStatus()) { case DELETED_BLOCK: removeStoredBlock(rdbi.getBlock(), node); deleted++; break; case RECEIVED_BLOCK: - addBlock(node, rdbi.getBlock(), rdbi.getDelHints()); + addBlock(node, srdb.getStorageID(), rdbi.getBlock(), rdbi.getDelHints()); received++; break; case RECEIVING_BLOCK: receiving++; - processAndHandleReportedBlock(node, rdbi.getBlock(), + processAndHandleReportedBlock(node, srdb.getStorageID(), rdbi.getBlock(), ReplicaState.RBW, null); break; default: diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index c542ae343e1..0c3ff98fc4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -18,15 +18,20 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Queue; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.util.Time; @@ -93,8 +98,9 @@ public class DatanodeDescriptor extends DatanodeInfo { } } - private volatile BlockInfo blockList = null; - private int numBlocks = 0; + private final Map storageMap = + new HashMap(); + // isAlive == heartbeats.contains(this) // This is an optimization, because contains takes O(n) time on Arraylist public boolean isAlive = false; @@ -217,39 +223,47 @@ public class DatanodeDescriptor extends DatanodeInfo { } /** - * Add datanode to the block. - * Add block to the head of the list of blocks belonging to the data-node. + * Add data-node to the block. Add block to the head of the list of blocks + * belonging to the data-node. */ - public boolean addBlock(BlockInfo b) { - if(!b.addNode(this)) - return false; - // add to the head of the data-node list - blockList = b.listInsert(blockList, this); - numBlocks++; - return true; - } - - /** - * Remove block from the list of blocks belonging to the data-node. - * Remove datanode from the block. - */ - public boolean removeBlock(BlockInfo b) { - blockList = b.listRemove(blockList, this); - if ( b.removeNode(this) ) { - numBlocks--; - return true; - } else { - return false; + public boolean addBlock(String storageID, BlockInfo b) { + DatanodeStorageInfo s = getStorageInfo(storageID); + if (s != null) { + return s.addBlock(b); } + return false; + } + + DatanodeStorageInfo getStorageInfo(String storageID) { + return storageMap.get(storageID); + } + public Collection getStorageInfos() { + return storageMap.values(); } /** - * Move block to the head of the list of blocks belonging to the data-node. - * @return the index of the head of the blockList + * Remove block from the list of blocks belonging to the data-node. Remove + * data-node from the block. */ - int moveBlockToHead(BlockInfo b, int curIndex, int headIndex) { - blockList = b.moveBlockToHead(blockList, this, curIndex, headIndex); - return curIndex; + boolean removeBlock(BlockInfo b) { + int index = b.findStorageInfo(this); + DatanodeStorageInfo s = b.getStorageInfo(index); + if (s != null) { + return s.removeBlock(b); + } + return false; + } + + /** + * Remove block from the list of blocks belonging to the data-node. Remove + * data-node from the block. + */ + boolean removeBlock(String storageID, BlockInfo b) { + DatanodeStorageInfo s = getStorageInfo(storageID); + if (s != null) { + return s.removeBlock(b); + } + return false; } /** @@ -257,7 +271,7 @@ public class DatanodeDescriptor extends DatanodeInfo { * @return the head of the blockList */ protected BlockInfo getHead(){ - return blockList; + return getBlockIterator().next(); } /** @@ -268,9 +282,12 @@ public class DatanodeDescriptor extends DatanodeInfo { * @return the new block */ public BlockInfo replaceBlock(BlockInfo oldBlock, BlockInfo newBlock) { - boolean done = removeBlock(oldBlock); + int index = oldBlock.findStorageInfo(this); + DatanodeStorageInfo s = oldBlock.getStorageInfo(index); + boolean done = s.removeBlock(oldBlock); assert done : "Old block should belong to the data-node when replacing"; - done = addBlock(newBlock); + + done = s.addBlock(newBlock); assert done : "New block should not belong to the data-node when replacing"; return newBlock; } @@ -281,7 +298,6 @@ public class DatanodeDescriptor extends DatanodeInfo { setBlockPoolUsed(0); setDfsUsed(0); setXceiverCount(0); - this.blockList = null; this.invalidateBlocks.clear(); this.volumeFailures = 0; } @@ -295,7 +311,12 @@ public class DatanodeDescriptor extends DatanodeInfo { } public int numBlocks() { - return numBlocks; + // TODO: synchronization + int blocks = 0; + for (DatanodeStorageInfo entry : storageMap.values()) { + blocks += entry.numBlocks(); + } + return blocks; } /** @@ -314,38 +335,52 @@ public class DatanodeDescriptor extends DatanodeInfo { rollBlocksScheduled(getLastUpdate()); } - /** - * Iterates over the list of blocks belonging to the datanode. - */ - public static class BlockIterator implements Iterator { - private BlockInfo current; - private DatanodeDescriptor node; - - BlockIterator(BlockInfo head, DatanodeDescriptor dn) { - this.current = head; - this.node = dn; + private static class BlockIterator implements Iterator { + private final int maxIndex; + private int index = 0; + private List> iterators = new ArrayList>(); + + private BlockIterator(final Iterable storages) { + for (DatanodeStorageInfo e : storages) { + iterators.add(e.getBlockIterator()); + } + maxIndex = iterators.size() - 1; + } + + private BlockIterator(final DatanodeStorageInfo storage) { + iterators.add(storage.getBlockIterator()); + maxIndex = iterators.size() - 1; } @Override public boolean hasNext() { - return current != null; + update(); + return iterators.get(index).hasNext(); } @Override public BlockInfo next() { - BlockInfo res = current; - current = current.getNext(current.findDatanode(node)); - return res; + update(); + return iterators.get(index).next(); } - + @Override - public void remove() { - throw new UnsupportedOperationException("Sorry. can't remove."); + public void remove() { + throw new UnsupportedOperationException("Remove unsupported."); + } + + private void update() { + while(index < maxIndex && !iterators.get(index).hasNext()) { + index++; + } } } - public Iterator getBlockIterator() { - return new BlockIterator(this.blockList, this); + Iterator getBlockIterator() { + return new BlockIterator(storageMap.values()); + } + Iterator getBlockIterator(final String storageID) { + return new BlockIterator(storageMap.get(storageID)); } /** @@ -601,4 +636,15 @@ public class DatanodeDescriptor extends DatanodeInfo { } return sb.toString(); } + + DatanodeStorageInfo updateStorage(DatanodeStorage s) { + DatanodeStorageInfo storage = getStorageInfo(s.getStorageID()); + if (storage == null) { + storage = new DatanodeStorageInfo(this, s); + storageMap.put(s.getStorageID(), storage); + } else { + storage.setState(s.getState()); + } + return storage; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java new file mode 100644 index 00000000000..805e334acbf --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.blockmanagement; + +import java.util.Iterator; + +import org.apache.hadoop.hdfs.StorageType; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; + +/** + * A Datanode has one or more storages. A storage in the Datanode is represented + * by this class. + */ +public class DatanodeStorageInfo { + /** + * Iterates over the list of blocks belonging to the data-node. + */ + static class BlockIterator implements Iterator { + private BlockInfo current; + private DatanodeStorageInfo node; + + BlockIterator(BlockInfo head, DatanodeStorageInfo dn) { + this.current = head; + this.node = dn; + } + + public boolean hasNext() { + return current != null; + } + + public BlockInfo next() { + BlockInfo res = current; + current = current.getNext(current.findStorageInfo(node)); + return res; + } + + public void remove() { + throw new UnsupportedOperationException("Sorry. can't remove."); + } + } + + private final DatanodeDescriptor dn; + private final String storageID; + private final StorageType storageType; + private State state; + private long capacity; + private long dfsUsed; + private long remaining; + private volatile BlockInfo blockList = null; + + DatanodeStorageInfo(DatanodeDescriptor dn, DatanodeStorage s) { + this.dn = dn; + this.storageID = s.getStorageID(); + this.storageType = s.getStorageType(); + this.state = s.getState(); + } + + public void setUtilization(long capacity, long dfsUsed, long remaining) { + this.capacity = capacity; + this.dfsUsed = dfsUsed; + this.remaining = remaining; + } + + public void setState(State s) { + this.state = s; + + // TODO: if goes to failed state cleanup the block list + } + + public State getState() { + return this.state; + } + + public String getStorageID() { + return storageID; + } + + public long getCapacity() { + return capacity; + } + + public long getDfsUsed() { + return dfsUsed; + } + + public long getRemaining() { + return remaining; + } + + public boolean addBlock(BlockInfo b) { + if(!b.addStorage(this)) + return false; + // add to the head of the data-node list + blockList = b.listInsert(blockList, this); + return true; + } + + public boolean removeBlock(BlockInfo b) { + blockList = b.listRemove(blockList, this); + return b.removeStorage(this); + } + + public int numBlocks() { + return blockList == null ? 0 : blockList.listCount(this); + } + + Iterator getBlockIterator() { + return new BlockIterator(this.blockList, this); + } + + public void updateState(StorageReport r) { + capacity = r.getCapacity(); + dfsUsed = r.getDfsUsed(); + remaining = r.getRemaining(); + } + + public DatanodeDescriptor getDatanodeDescriptor() { + return dn; + } + + @Override + public String toString() { + return "[" + storageType + "]" + storageID + ":" + state; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java index 860d1d261f8..8afd3ce6ba7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java @@ -42,11 +42,13 @@ class PendingDataNodeMessages { static class ReportedBlockInfo { private final Block block; private final DatanodeDescriptor dn; + private final String storageID; private final ReplicaState reportedState; - ReportedBlockInfo(DatanodeDescriptor dn, Block block, + ReportedBlockInfo(DatanodeDescriptor dn, String storageID, Block block, ReplicaState reportedState) { this.dn = dn; + this.storageID = storageID; this.block = block; this.reportedState = reportedState; } @@ -58,6 +60,10 @@ class PendingDataNodeMessages { DatanodeDescriptor getNode() { return dn; } + + String getStorageID() { + return storageID; + } ReplicaState getReportedState() { return reportedState; @@ -70,11 +76,11 @@ class PendingDataNodeMessages { } } - void enqueueReportedBlock(DatanodeDescriptor dn, Block block, + void enqueueReportedBlock(DatanodeDescriptor dn, String storageID, Block block, ReplicaState reportedState) { block = new Block(block); getBlockQueue(block).add( - new ReportedBlockInfo(dn, block, reportedState)); + new ReportedBlockInfo(dn, storageID, block, reportedState)); count++; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 030893028c4..d83d53a5f35 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -162,7 +162,13 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; -import org.apache.hadoop.hdfs.server.blockmanagement.*; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStatistics; +import org.apache.hadoop.hdfs.server.blockmanagement.OutOfV1GenerationStampsException; import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; @@ -174,14 +180,7 @@ import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.JournalSet.JournalAndStream; import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; -import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; -import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; -import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter; -import org.apache.hadoop.hdfs.server.namenode.startupprogress.Status; -import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step; -import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType; import org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer; import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; import org.apache.hadoop.hdfs.server.namenode.ha.HAState; @@ -193,6 +192,12 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottab import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileWithSnapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotManager; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.Status; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; @@ -201,7 +206,7 @@ import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat; import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; -import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; +import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RetryCache; @@ -3785,7 +3790,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, // Otherwise fsck will report these blocks as MISSING, especially if the // blocksReceived from Datanodes take a long time to arrive. for (int i = 0; i < descriptors.length; i++) { - descriptors[i].addBlock(storedBlock); + descriptors[i].addBlock(newtargetstorages[i], storedBlock); } } // add pipeline locations into the INodeUnderConstruction @@ -5088,11 +5093,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } public void processIncrementalBlockReport(final DatanodeID nodeID, - final String poolId, final ReceivedDeletedBlockInfo blockInfos[]) + final String poolId, final StorageReceivedDeletedBlocks srdb) throws IOException { writeLock(); try { - blockManager.processIncrementalBlockReport(nodeID, poolId, blockInfos); + blockManager.processIncrementalBlockReport(nodeID, poolId, srdb); } finally { writeUnlock(); } @@ -5578,8 +5583,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, ExtendedBlock blk = blocks[i].getBlock(); DatanodeInfo[] nodes = blocks[i].getLocations(); for (int j = 0; j < nodes.length; j++) { - DatanodeInfo dn = nodes[j]; - blockManager.findAndMarkBlockAsCorrupt(blk, dn, + //TODO: add "storageID to LocatedBlock + blockManager.findAndMarkBlockAsCorrupt(blk, nodes[j], "STORAGE_ID", "client machine reported it"); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 0a3309fdbb4..62a17a96f5b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -42,8 +42,8 @@ import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Options; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.ParentNotDirectoryException; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; @@ -59,21 +59,21 @@ import org.apache.hadoop.hdfs.HDFSPolicyProvider; import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; +import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException; import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; @@ -89,9 +89,10 @@ import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolServerSideTranslatorPB; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.IncorrectVersionException; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; @@ -943,14 +944,18 @@ class NameNodeRpcServer implements NamenodeProtocols { public DatanodeCommand blockReport(DatanodeRegistration nodeReg, String poolId, StorageBlockReport[] reports) throws IOException { verifyRequest(nodeReg); - BlockListAsLongs blist = new BlockListAsLongs(reports[0].getBlocks()); if(blockStateChangeLog.isDebugEnabled()) { blockStateChangeLog.debug("*BLOCK* NameNode.blockReport: " - + "from " + nodeReg + " " + blist.getNumberOfBlocks() - + " blocks"); + + "from " + nodeReg + ", reports.length=" + reports.length); + } + final BlockManager bm = namesystem.getBlockManager(); + for(StorageBlockReport r : reports) { + final BlockListAsLongs blocks = new BlockListAsLongs(r.getBlocks()); + bm.processReport(nodeReg, r.getStorage(), poolId, blocks); } - namesystem.getBlockManager().processReport(nodeReg, poolId, blist); + DatanodeDescriptor datanode = bm.getDatanodeManager().getDatanode(nodeReg); + datanode.receivedBlockReport(); if (nn.getFSImage().isUpgradeFinalized() && !nn.isStandbyState()) return new FinalizeCommand(poolId); return null; @@ -965,8 +970,9 @@ class NameNodeRpcServer implements NamenodeProtocols { +"from "+nodeReg+" "+receivedAndDeletedBlocks.length +" blocks."); } - namesystem.processIncrementalBlockReport( - nodeReg, poolId, receivedAndDeletedBlocks[0].getBlocks()); + for(StorageReceivedDeletedBlocks r : receivedAndDeletedBlocks) { + namesystem.processIncrementalBlockReport(nodeReg, poolId, r); + } } @Override // DatanodeProtocol diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java index 458880af566..a829096c32b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java @@ -157,7 +157,7 @@ public class TestFileCorruption { ns.writeLock(); try { cluster.getNamesystem().getBlockManager().findAndMarkBlockAsCorrupt( - blk, new DatanodeInfo(dnR), "TEST"); + blk, new DatanodeInfo(dnR), "TEST", "STORAGE_ID"); } finally { ns.writeUnlock(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java deleted file mode 100644 index 8b7111f010e..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.blockmanagement; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.Random; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.server.common.GenerationStamp; -import org.junit.Test; - -/** - * This class provides tests for BlockInfo class, which is used in BlocksMap. - * The test covers BlockList.listMoveToHead, used for faster block report - * processing in DatanodeDescriptor.reportDiff. - */ - -public class TestBlockInfo { - - private static final Log LOG = LogFactory - .getLog("org.apache.hadoop.hdfs.TestBlockInfo"); - - @Test - public void testBlockListMoveToHead() throws Exception { - LOG.info("BlockInfo moveToHead tests..."); - - final int MAX_BLOCKS = 10; - - DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor(); - ArrayList blockList = new ArrayList(MAX_BLOCKS); - ArrayList blockInfoList = new ArrayList(); - int headIndex; - int curIndex; - - LOG.info("Building block list..."); - for (int i = 0; i < MAX_BLOCKS; i++) { - blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP)); - blockInfoList.add(new BlockInfo(blockList.get(i), 3)); - dd.addBlock(blockInfoList.get(i)); - - // index of the datanode should be 0 - assertEquals("Find datanode should be 0", 0, blockInfoList.get(i) - .findDatanode(dd)); - } - - // list length should be equal to the number of blocks we inserted - LOG.info("Checking list length..."); - assertEquals("Length should be MAX_BLOCK", MAX_BLOCKS, dd.numBlocks()); - Iterator it = dd.getBlockIterator(); - int len = 0; - while (it.hasNext()) { - it.next(); - len++; - } - assertEquals("There should be MAX_BLOCK blockInfo's", MAX_BLOCKS, len); - - headIndex = dd.getHead().findDatanode(dd); - - LOG.info("Moving each block to the head of the list..."); - for (int i = 0; i < MAX_BLOCKS; i++) { - curIndex = blockInfoList.get(i).findDatanode(dd); - headIndex = dd.moveBlockToHead(blockInfoList.get(i), curIndex, headIndex); - // the moved element must be at the head of the list - assertEquals("Block should be at the head of the list now.", - blockInfoList.get(i), dd.getHead()); - } - - // move head of the list to the head - this should not change the list - LOG.info("Moving head to the head..."); - - BlockInfo temp = dd.getHead(); - curIndex = 0; - headIndex = 0; - dd.moveBlockToHead(temp, curIndex, headIndex); - assertEquals( - "Moving head to the head of the list shopuld not change the list", - temp, dd.getHead()); - - // check all elements of the list against the original blockInfoList - LOG.info("Checking elements of the list..."); - temp = dd.getHead(); - assertNotNull("Head should not be null", temp); - int c = MAX_BLOCKS - 1; - while (temp != null) { - assertEquals("Expected element is not on the list", - blockInfoList.get(c--), temp); - temp = temp.getNext(0); - } - - LOG.info("Moving random blocks to the head of the list..."); - headIndex = dd.getHead().findDatanode(dd); - Random rand = new Random(); - for (int i = 0; i < MAX_BLOCKS; i++) { - int j = rand.nextInt(MAX_BLOCKS); - curIndex = blockInfoList.get(j).findDatanode(dd); - headIndex = dd.moveBlockToHead(blockInfoList.get(j), curIndex, headIndex); - // the moved element must be at the head of the list - assertEquals("Block should be at the head of the list now.", - blockInfoList.get(j), dd.getHead()); - } - } -} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index e88ec92e39b..67306cf782e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -124,7 +124,7 @@ public class TestBlockManager { private void doBasicTest(int testIndex) { List origNodes = getNodes(0, 1); - BlockInfo blockInfo = addBlockOnNodes((long)testIndex, origNodes); + BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes); DatanodeDescriptor[] pipeline = scheduleSingleReplication(blockInfo); assertEquals(2, pipeline.length); @@ -307,7 +307,7 @@ public class TestBlockManager { private void doTestSufficientlyReplBlocksUsesNewRack(int testIndex) { // Originally on only nodes in rack A. List origNodes = rackA; - BlockInfo blockInfo = addBlockOnNodes((long)testIndex, origNodes); + BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes); DatanodeDescriptor pipeline[] = scheduleSingleReplication(blockInfo); assertEquals(2, pipeline.length); // single new copy @@ -340,7 +340,7 @@ public class TestBlockManager { List origNodes) throws Exception { assertEquals(0, bm.numOfUnderReplicatedBlocks()); - addBlockOnNodes((long)testIndex, origNodes); + addBlockOnNodes(testIndex, origNodes); bm.processMisReplicatedBlocks(); assertEquals(0, bm.numOfUnderReplicatedBlocks()); } @@ -353,7 +353,7 @@ public class TestBlockManager { private void fulfillPipeline(BlockInfo blockInfo, DatanodeDescriptor[] pipeline) throws IOException { for (int i = 1; i < pipeline.length; i++) { - bm.addBlock(pipeline[i], blockInfo, null); + bm.addBlock(pipeline[i], "STORAGE_ID", blockInfo, null); } } @@ -362,7 +362,9 @@ public class TestBlockManager { BlockInfo blockInfo = new BlockInfo(block, 3); for (DatanodeDescriptor dn : nodes) { - blockInfo.addNode(dn); + for (DatanodeStorageInfo storage : dn.getStorageInfos()) { + blockInfo.addStorage(storage); + } } return blockInfo; } @@ -508,12 +510,12 @@ public class TestBlockManager { assertTrue(node.isFirstBlockReport()); // send block report, should be processed reset(node); - bm.processReport(node, "pool", new BlockListAsLongs(null, null)); + bm.processReport(node, null, "pool", new BlockListAsLongs(null, null)); verify(node).receivedBlockReport(); assertFalse(node.isFirstBlockReport()); // send block report again, should NOT be processed reset(node); - bm.processReport(node, "pool", new BlockListAsLongs(null, null)); + bm.processReport(node, null, "pool", new BlockListAsLongs(null, null)); verify(node, never()).receivedBlockReport(); assertFalse(node.isFirstBlockReport()); @@ -525,7 +527,7 @@ public class TestBlockManager { assertTrue(node.isFirstBlockReport()); // ready for report again // send block report, should be processed after restart reset(node); - bm.processReport(node, "pool", new BlockListAsLongs(null, null)); + bm.processReport(node, null, "pool", new BlockListAsLongs(null, null)); verify(node).receivedBlockReport(); assertFalse(node.isFirstBlockReport()); } @@ -550,7 +552,7 @@ public class TestBlockManager { // send block report while pretending to already have blocks reset(node); doReturn(1).when(node).numBlocks(); - bm.processReport(node, "pool", new BlockListAsLongs(null, null)); + bm.processReport(node, null, "pool", new BlockListAsLongs(null, null)); verify(node).receivedBlockReport(); assertFalse(node.isFirstBlockReport()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java index bd030fb94dd..41afa739d65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java @@ -59,17 +59,18 @@ public class TestDatanodeDescriptor { assertEquals(0, dd.numBlocks()); BlockInfo blk = new BlockInfo(new Block(1L), 1); BlockInfo blk1 = new BlockInfo(new Block(2L), 2); + final String storageID = "STORAGE_ID"; // add first block - assertTrue(dd.addBlock(blk)); + assertTrue(dd.addBlock(storageID, blk)); assertEquals(1, dd.numBlocks()); // remove a non-existent block assertFalse(dd.removeBlock(blk1)); assertEquals(1, dd.numBlocks()); // add an existent block - assertFalse(dd.addBlock(blk)); + assertFalse(dd.addBlock(storageID, blk)); assertEquals(1, dd.numBlocks()); // add second block - assertTrue(dd.addBlock(blk1)); + assertTrue(dd.addBlock(storageID, blk1)); assertEquals(2, dd.numBlocks()); // remove first block assertTrue(dd.removeBlock(blk)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java index 3c7ad8ca021..dbff77402ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java @@ -43,8 +43,8 @@ public class TestPendingDataNodeMessages { @Test public void testQueues() { DatanodeDescriptor fakeDN = DFSTestUtil.getLocalDatanodeDescriptor(); - msgs.enqueueReportedBlock(fakeDN, block1Gs1, ReplicaState.FINALIZED); - msgs.enqueueReportedBlock(fakeDN, block1Gs2, ReplicaState.FINALIZED); + msgs.enqueueReportedBlock(fakeDN, "STORAGE_ID", block1Gs1, ReplicaState.FINALIZED); + msgs.enqueueReportedBlock(fakeDN, "STORAGE_ID", block1Gs2, ReplicaState.FINALIZED); assertEquals(2, msgs.count()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java index 0c1fee98646..8eb9ced9e2f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java @@ -291,9 +291,9 @@ public class TestPendingReplication { cluster.getNamesystem().writeLock(); try { bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0], - "TEST"); + "STORAGE_ID", "TEST"); bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[1], - "TEST"); + "STORAGE_ID", "TEST"); } finally { cluster.getNamesystem().writeUnlock(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index ba6c3737266..547e8536f2b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -1096,7 +1096,7 @@ public class TestReplicationPolicy { // Adding this block will increase its current replication, and that will // remove it from the queue. bm.addStoredBlockUnderConstruction(info, - TestReplicationPolicy.dataNodes[0], ReplicaState.FINALIZED); + TestReplicationPolicy.dataNodes[0], "STORAGE", ReplicaState.FINALIZED); // Choose 1 block from UnderReplicatedBlocks. Then it should pick 1 block // from QUEUE_VERY_UNDER_REPLICATED. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index 007fda5fb5e..d8aed466414 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -235,7 +235,7 @@ public class TestNameNodeMetrics { cluster.getNamesystem().writeLock(); try { bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0], - "TEST"); + "STORAGE_ID", "TEST"); } finally { cluster.getNamesystem().writeUnlock(); } @@ -286,7 +286,7 @@ public class TestNameNodeMetrics { cluster.getNamesystem().writeLock(); try { bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0], - "TEST"); + "STORAGE_ID", "TEST"); } finally { cluster.getNamesystem().writeUnlock(); } From b305f1ff081bf6fed2abf7f5115af9fbee2f00f7 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 29 Aug 2013 00:21:08 +0000 Subject: [PATCH 07/76] Update CHANGES_HDFS-2832.txt with HDFS-4985, HDFS-5115 and HDFS-5000. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1518426 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 7ab7c4c1650..8bb94ac6f6f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -1,4 +1,14 @@ BREAKDOWN OF HDFS-2832 ENABLE SUPPORT FOR HETEROGENEOUS STORAGES IN HDFS +IMPROVEMENTS: + + HDFS-4985. Add storage type to the protocol and expose it in block report + and block locations. (Arpit Agarwal) + + HDFS-5115. Make StorageID a UUID. (Arpit Agarwal) + + HDFS-5000. DataNode configuration should allow specifying storage type. + (Arpit Agarwal) + HDFS-4987. Namenode changes to track multiple storages per datanode. (szetszwo) From f7e3bc553d770ea340ac6ed8ed95e5bd1ebf7383 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 3 Sep 2013 04:04:22 +0000 Subject: [PATCH 08/76] HDFS-5154. Fix TestBlockManager and TestDatanodeDescriptor after HDFS-4987. Contributed by Junping Du git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1519548 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 + .../blockmanagement/DatanodeDescriptor.java | 9 ++- .../blockmanagement/BlockManagerTestUtil.java | 21 ++++++ .../blockmanagement/TestBlockManager.java | 67 +++++++++++++------ .../TestDatanodeDescriptor.java | 7 +- 5 files changed, 80 insertions(+), 27 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 8bb94ac6f6f..916aace1530 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -12,3 +12,6 @@ IMPROVEMENTS: HDFS-4987. Namenode changes to track multiple storages per datanode. (szetszwo) + + HDFS-5154. Fix TestBlockManager and TestDatanodeDescriptor after HDFS-4987. + (Junping Du via szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 0c3ff98fc4e..bc1107beb73 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -247,9 +247,12 @@ public class DatanodeDescriptor extends DatanodeInfo { */ boolean removeBlock(BlockInfo b) { int index = b.findStorageInfo(this); - DatanodeStorageInfo s = b.getStorageInfo(index); - if (s != null) { - return s.removeBlock(b); + // if block exists on this datanode + if (index >= 0) { + DatanodeStorageInfo s = b.getStorageInfo(index); + if (s != null) { + return s.removeBlock(b); + } } return false; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java index 35639fd0763..006ee42bb58 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java @@ -24,9 +24,11 @@ import java.util.Iterator; import java.util.Set; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.util.Daemon; import org.junit.Assert; @@ -215,4 +217,23 @@ public class BlockManagerTestUtil { public static void checkHeartbeat(BlockManager bm) { bm.getDatanodeManager().getHeartbeatManager().heartbeatCheck(); } + + public static DatanodeDescriptor getLocalDatanodeDescriptor( + boolean initializeStorage) { + DatanodeDescriptor dn = new DatanodeDescriptor(DFSTestUtil.getLocalDatanodeID()); + if (initializeStorage) { + dn.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); + } + return dn; + } + + public static DatanodeDescriptor getDatanodeDescriptor(String ipAddr, + String rackLocation, boolean initializeStorage) { + DatanodeDescriptor dn = DFSTestUtil.getDatanodeDescriptor(ipAddr, + DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT, rackLocation); + if (initializeStorage) { + dn.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); + } + return dn; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 67306cf782e..1ddb91cfe1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.ArrayList; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map.Entry; @@ -39,6 +40,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.net.NetworkTopology; import org.junit.Before; import org.junit.Test; @@ -80,17 +82,17 @@ public class TestBlockManager { Mockito.doReturn(true).when(fsn).hasWriteLock(); bm = new BlockManager(fsn, fsn, conf); nodes = ImmutableList.of( - DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/rackA"), - DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/rackA"), - DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/rackA"), - DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/rackB"), - DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/rackB"), - DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/rackB") - ); + BlockManagerTestUtil.getDatanodeDescriptor("1.1.1.1", "/rackA", true), + BlockManagerTestUtil.getDatanodeDescriptor("2.2.2.2", "/rackA", true), + BlockManagerTestUtil.getDatanodeDescriptor("3.3.3.3", "/rackA", true), + BlockManagerTestUtil.getDatanodeDescriptor("4.4.4.4", "/rackB", true), + BlockManagerTestUtil.getDatanodeDescriptor("5.5.5.5", "/rackB", true), + BlockManagerTestUtil.getDatanodeDescriptor("6.6.6.6", "/rackB", true) + ); rackA = nodes.subList(0, 3); rackB = nodes.subList(3, 6); } - + private void addNodes(Iterable nodesToAdd) { NetworkTopology cluster = bm.getDatanodeManager().getNetworkTopology(); // construct network topology @@ -282,6 +284,7 @@ public class TestBlockManager { // the third off-rack replica. DatanodeDescriptor rackCNode = DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/rackC"); + rackCNode.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); addNodes(ImmutableList.of(rackCNode)); try { DatanodeDescriptor[] pipeline2 = scheduleSingleReplication(blockInfo); @@ -322,15 +325,15 @@ public class TestBlockManager { @Test public void testBlocksAreNotUnderreplicatedInSingleRack() throws Exception { List nodes = ImmutableList.of( - DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/rackA"), - DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/rackA"), - DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/rackA"), - DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/rackA"), - DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/rackA"), - DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/rackA") + BlockManagerTestUtil.getDatanodeDescriptor("1.1.1.1", "/rackA", true), + BlockManagerTestUtil.getDatanodeDescriptor("2.2.2.2", "/rackA", true), + BlockManagerTestUtil.getDatanodeDescriptor("3.3.3.3", "/rackA", true), + BlockManagerTestUtil.getDatanodeDescriptor("4.4.4.4", "/rackA", true), + BlockManagerTestUtil.getDatanodeDescriptor("5.5.5.5", "/rackA", true), + BlockManagerTestUtil.getDatanodeDescriptor("6.6.6.6", "/rackA", true) ); addNodes(nodes); - List origNodes = nodes.subList(0, 3);; + List origNodes = nodes.subList(0, 3); for (int i = 0; i < NUM_TEST_ITERS; i++) { doTestSingleRackClusterIsSufficientlyReplicated(i, origNodes); } @@ -353,7 +356,17 @@ public class TestBlockManager { private void fulfillPipeline(BlockInfo blockInfo, DatanodeDescriptor[] pipeline) throws IOException { for (int i = 1; i < pipeline.length; i++) { - bm.addBlock(pipeline[i], "STORAGE_ID", blockInfo, null); + DatanodeDescriptor dn = pipeline[i]; + + Iterator iterator = dn.getStorageInfos().iterator(); + if (iterator.hasNext()) { + DatanodeStorageInfo storage = iterator.next(); + bm.addBlock(dn, storage.getStorageID(), blockInfo, null); + blockInfo.addStorage(storage); + } else { + throw new RuntimeException("Storage info on node: " + dn.getHostName() + + " is invalid."); + } } } @@ -494,7 +507,10 @@ public class TestBlockManager { @Test public void testSafeModeIBR() throws Exception { DatanodeDescriptor node = spy(nodes.get(0)); - node.setStorageID("dummy-storage"); + Iterator i = node.getStorageInfos().iterator(); + DatanodeStorageInfo ds = i.next(); + node.setStorageID(ds.getStorageID()); + node.isAlive = true; DatanodeRegistration nodeReg = @@ -510,12 +526,15 @@ public class TestBlockManager { assertTrue(node.isFirstBlockReport()); // send block report, should be processed reset(node); - bm.processReport(node, null, "pool", new BlockListAsLongs(null, null)); + + bm.processReport(node, new DatanodeStorage(ds.getStorageID()), "pool", + new BlockListAsLongs(null, null)); verify(node).receivedBlockReport(); assertFalse(node.isFirstBlockReport()); // send block report again, should NOT be processed reset(node); - bm.processReport(node, null, "pool", new BlockListAsLongs(null, null)); + bm.processReport(node, new DatanodeStorage(ds.getStorageID()), "pool", + new BlockListAsLongs(null, null)); verify(node, never()).receivedBlockReport(); assertFalse(node.isFirstBlockReport()); @@ -527,7 +546,8 @@ public class TestBlockManager { assertTrue(node.isFirstBlockReport()); // ready for report again // send block report, should be processed after restart reset(node); - bm.processReport(node, null, "pool", new BlockListAsLongs(null, null)); + bm.processReport(node, new DatanodeStorage(ds.getStorageID()), "pool", + new BlockListAsLongs(null, null)); verify(node).receivedBlockReport(); assertFalse(node.isFirstBlockReport()); } @@ -535,7 +555,9 @@ public class TestBlockManager { @Test public void testSafeModeIBRAfterIncremental() throws Exception { DatanodeDescriptor node = spy(nodes.get(0)); - node.setStorageID("dummy-storage"); + Iterator i = node.getStorageInfos().iterator(); + DatanodeStorageInfo ds = i.next(); + node.setStorageID(ds.getStorageID()); node.isAlive = true; DatanodeRegistration nodeReg = @@ -552,7 +574,8 @@ public class TestBlockManager { // send block report while pretending to already have blocks reset(node); doReturn(1).when(node).numBlocks(); - bm.processReport(node, null, "pool", new BlockListAsLongs(null, null)); + bm.processReport(node, new DatanodeStorage(ds.getStorageID()), "pool", + new BlockListAsLongs(null, null)); verify(node).receivedBlockReport(); assertFalse(node.isFirstBlockReport()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java index 41afa739d65..bf240d7d46d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.util.ArrayList; +import java.util.Iterator; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.protocol.Block; @@ -55,11 +56,13 @@ public class TestDatanodeDescriptor { @Test public void testBlocksCounter() throws Exception { - DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor(); + DatanodeDescriptor dd = BlockManagerTestUtil.getLocalDatanodeDescriptor(true); assertEquals(0, dd.numBlocks()); BlockInfo blk = new BlockInfo(new Block(1L), 1); BlockInfo blk1 = new BlockInfo(new Block(2L), 2); - final String storageID = "STORAGE_ID"; + Iterator iterator = dd.getStorageInfos().iterator(); + assertTrue(iterator.hasNext()); + final String storageID = iterator.next().getStorageID(); // add first block assertTrue(dd.addBlock(storageID, blk)); assertEquals(1, dd.numBlocks()); From 3f070e83b1f4e0211ece8c0ab508a61188ad352a Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 3 Sep 2013 14:03:35 +0000 Subject: [PATCH 09/76] HDFS-5009. Include storage information in the LocatedBlock. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1519691 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 4 +- .../apache/hadoop/hdfs/DFSOutputStream.java | 6 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 2 +- .../hadoop/hdfs/protocol/LocatedBlock.java | 24 +++++- ...amenodeProtocolServerSideTranslatorPB.java | 10 ++- .../ClientNamenodeProtocolTranslatorPB.java | 3 +- .../hadoop/hdfs/protocolPB/PBHelper.java | 3 + .../server/blockmanagement/BlockInfo.java | 6 +- .../BlockInfoUnderConstruction.java | 37 ++++---- .../server/blockmanagement/BlockManager.java | 86 ++++++++----------- .../server/blockmanagement/BlocksMap.java | 25 +++--- .../blockmanagement/DatanodeManager.java | 53 +++++++++--- .../blockmanagement/DatanodeStorageInfo.java | 20 ++++- .../MutableBlockCollection.java | 2 +- .../hdfs/server/namenode/FSDirectory.java | 3 +- .../hdfs/server/namenode/FSNamesystem.java | 64 +++++++------- .../namenode/INodeFileUnderConstruction.java | 3 +- .../server/namenode/NameNodeRpcServer.java | 4 +- .../server/namenode/NamenodeJspHelper.java | 6 +- .../main/proto/ClientNamenodeProtocol.proto | 1 + .../hadoop-hdfs/src/main/proto/hdfs.proto | 1 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 11 ++- .../blockmanagement/BlockManagerTestUtil.java | 5 +- .../TestBlockInfoUnderConstruction.java | 15 ++-- .../TestHeartbeatHandling.java | 14 +-- .../server/blockmanagement/TestNodeCount.java | 6 +- .../TestReplicationPolicy.java | 9 +- .../TestUnderReplicatedBlocks.java | 3 +- .../TestCommitBlockSynchronization.java | 4 +- .../namenode/ha/TestPipelinesFailover.java | 18 ++-- .../namenode/ha/TestRetryCacheWithHA.java | 7 +- 31 files changed, 269 insertions(+), 186 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 916aace1530..5a6f2c76c4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -13,5 +13,7 @@ IMPROVEMENTS: HDFS-4987. Namenode changes to track multiple storages per datanode. (szetszwo) - HDFS-5154. Fix TestBlockManager and TestDatanodeDescriptor after HDFS-4987. + HDFS-5154. Fix TestBlockManager and TestDatanodeDescriptor after HDFS-4987. (Junping Du via szetszwo) + + HDFS-5009. Include storage information in the LocatedBlock. (szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 927c530556d..a9bcc16eb44 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -85,7 +85,6 @@ import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Time; -import org.mortbay.log.Log; import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.CacheBuilder; @@ -312,6 +311,8 @@ public class DFSOutputStream extends FSOutputSummer private DataInputStream blockReplyStream; private ResponseProcessor response = null; private volatile DatanodeInfo[] nodes = null; // list of targets for current block + //TODO: update storage IDs + private volatile String[] storageIDs = null; private LoadingCache excludedNodes = CacheBuilder.newBuilder() .expireAfterWrite( @@ -1039,7 +1040,8 @@ public class DFSOutputStream extends FSOutputSummer // update pipeline at the namenode ExtendedBlock newBlock = new ExtendedBlock( block.getBlockPoolId(), block.getBlockId(), block.getNumBytes(), newGS); - dfsClient.namenode.updatePipeline(dfsClient.clientName, block, newBlock, nodes); + dfsClient.namenode.updatePipeline(dfsClient.clientName, block, newBlock, + nodes, storageIDs); // update client side generation stamp block = newBlock; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 5789c3615eb..66b6fbb0d90 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -982,7 +982,7 @@ public interface ClientProtocol { */ @AtMostOnce public void updatePipeline(String clientName, ExtendedBlock oldBlock, - ExtendedBlock newBlock, DatanodeID[] newNodes) + ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs) throws IOException; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java index 93724d7569e..e26ca81d073 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java @@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.security.token.Token; /** @@ -35,6 +36,8 @@ public class LocatedBlock { private ExtendedBlock b; private long offset; // offset of the first byte of the block in the file private DatanodeInfo[] locs; + /** Storage ID for each replica */ + private String[] storageIDs; // Storage type for each replica, if reported. private StorageType[] storageTypes; // corrupt flag is true if all of the replicas of a block are corrupt. @@ -53,10 +56,22 @@ public class LocatedBlock { public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, long startOffset, boolean corrupt) { - this(b, locs, null, startOffset, corrupt); + this(b, locs, null, null, startOffset, corrupt); } - public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, + public static LocatedBlock createLocatedBlock(ExtendedBlock b, + DatanodeStorageInfo[] storages, long startOffset, boolean corrupt) { + final DatanodeInfo[] locs = new DatanodeInfo[storages.length]; + final String[] storageIDs = new String[storages.length]; + final StorageType[] storageType = new StorageType[storages.length]; + for(int i = 0; i < storages.length; i++) { + locs[i] = storages[i].getDatanodeDescriptor(); + storageIDs[i] = storages[i].getStorageID(); + storageType[i] = storages[i].getStorageType(); + } + return new LocatedBlock(b, locs, storageIDs, storageType, startOffset, corrupt); + } + public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, String[] storageIDs, StorageType[] storageTypes, long startOffset, boolean corrupt) { this.b = b; @@ -67,6 +82,7 @@ public class LocatedBlock { } else { this.locs = locs; } + this.storageIDs = storageIDs; this.storageTypes = storageTypes; } @@ -94,6 +110,10 @@ public class LocatedBlock { return storageTypes; } + public String[] getStorageIDs() { + return storageIDs; + } + public long getStartOffset() { return offset; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index d7a18a60ac8..76ba6cab59a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -813,10 +813,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements UpdatePipelineRequestProto req) throws ServiceException { try { List newNodes = req.getNewNodesList(); - server - .updatePipeline(req.getClientName(), PBHelper.convert(req - .getOldBlock()), PBHelper.convert(req.getNewBlock()), PBHelper - .convert(newNodes.toArray(new DatanodeIDProto[newNodes.size()]))); + List newStorageIDs = req.getStorageIDsList(); + server.updatePipeline(req.getClientName(), + PBHelper.convert(req.getOldBlock()), + PBHelper.convert(req.getNewBlock()), + PBHelper.convert(newNodes.toArray(new DatanodeIDProto[newNodes.size()])), + newStorageIDs.toArray(new String[newStorageIDs.size()])); return VOID_UPDATEPIPELINE_RESPONSE; } catch (IOException e) { throw new ServiceException(e); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index b5be61a3bcd..7bcd014372f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -780,12 +780,13 @@ public class ClientNamenodeProtocolTranslatorPB implements @Override public void updatePipeline(String clientName, ExtendedBlock oldBlock, - ExtendedBlock newBlock, DatanodeID[] newNodes) throws IOException { + ExtendedBlock newBlock, DatanodeID[] newNodes, String[] storageIDs) throws IOException { UpdatePipelineRequestProto req = UpdatePipelineRequestProto.newBuilder() .setClientName(clientName) .setOldBlock(PBHelper.convert(oldBlock)) .setNewBlock(PBHelper.convert(newBlock)) .addAllNewNodes(Arrays.asList(PBHelper.convert(newNodes))) + .addAllStorageIDs(Arrays.asList(storageIDs)) .build(); try { rpcProxy.updatePipeline(null, req); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 3f015bdb09b..c82df6f8290 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -575,6 +575,8 @@ public class PBHelper { builder.addStorageTypes(PBHelper.convertStorageType(storageTypes[i])); } } + builder.addAllStorageIDs(Arrays.asList(b.getStorageIDs())); + return builder.setB(PBHelper.convert(b.getBlock())) .setBlockToken(PBHelper.convert(b.getBlockToken())) .setCorrupt(b.isCorrupt()).setOffset(b.getStartOffset()).build(); @@ -602,6 +604,7 @@ public class PBHelper { } LocatedBlock lb = new LocatedBlock(PBHelper.convert(proto.getB()), targets, + proto.getStorageIDsList().toArray(new String[proto.getStorageIDsCount()]), storageTypes, proto.getOffset(), proto.getCorrupt()); lb.setBlockToken(PBHelper.convert(proto.getBlockToken())); return lb; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index 7aa7721a022..9f7aabd4668 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -356,10 +356,10 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { * @return BlockInfoUnderConstruction - an under construction block. */ public BlockInfoUnderConstruction convertToBlockUnderConstruction( - BlockUCState s, DatanodeDescriptor[] targets) { + BlockUCState s, DatanodeStorageInfo[] targets) { if(isComplete()) { - return new BlockInfoUnderConstruction( - this, getBlockCollection().getBlockReplication(), s, targets); + return new BlockInfoUnderConstruction(this, + getBlockCollection().getBlockReplication(), s, targets); } // the block is already under construction BlockInfoUnderConstruction ucBlock = (BlockInfoUnderConstruction)this; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java index fc59acf10a6..e580c542969 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -64,12 +63,12 @@ public class BlockInfoUnderConstruction extends BlockInfo { * corresponding replicas. */ static class ReplicaUnderConstruction extends Block { - private DatanodeDescriptor expectedLocation; + private final DatanodeStorageInfo expectedLocation; private ReplicaState state; private boolean chosenAsPrimary; ReplicaUnderConstruction(Block block, - DatanodeDescriptor target, + DatanodeStorageInfo target, ReplicaState state) { super(block); this.expectedLocation = target; @@ -83,7 +82,7 @@ public class BlockInfoUnderConstruction extends BlockInfo { * It is not guaranteed, but expected, that the data-node actually has * the replica. */ - DatanodeDescriptor getExpectedLocation() { + private DatanodeStorageInfo getExpectedStorageLocation() { return expectedLocation; } @@ -119,7 +118,7 @@ public class BlockInfoUnderConstruction extends BlockInfo { * Is data-node the replica belongs to alive. */ boolean isAlive() { - return expectedLocation.isAlive; + return expectedLocation.getDatanodeDescriptor().isAlive; } @Override // Block @@ -163,7 +162,7 @@ public class BlockInfoUnderConstruction extends BlockInfo { */ public BlockInfoUnderConstruction(Block blk, int replication, BlockUCState state, - DatanodeDescriptor[] targets) { + DatanodeStorageInfo[] targets) { super(blk, replication); assert getBlockUCState() != BlockUCState.COMPLETE : "BlockInfoUnderConstruction cannot be in COMPLETE state"; @@ -187,7 +186,7 @@ public class BlockInfoUnderConstruction extends BlockInfo { } /** Set expected locations */ - public void setExpectedLocations(DatanodeDescriptor[] targets) { + public void setExpectedLocations(DatanodeStorageInfo[] targets) { int numLocations = targets == null ? 0 : targets.length; this.replicas = new ArrayList(numLocations); for(int i = 0; i < numLocations; i++) @@ -199,12 +198,12 @@ public class BlockInfoUnderConstruction extends BlockInfo { * Create array of expected replica locations * (as has been assigned by chooseTargets()). */ - public DatanodeDescriptor[] getExpectedLocations() { + public DatanodeStorageInfo[] getExpectedStorageLocations() { int numLocations = replicas == null ? 0 : replicas.size(); - DatanodeDescriptor[] locations = new DatanodeDescriptor[numLocations]; + DatanodeStorageInfo[] storages = new DatanodeStorageInfo[numLocations]; for(int i = 0; i < numLocations; i++) - locations[i] = replicas.get(i).getExpectedLocation(); - return locations; + storages[i] = replicas.get(i).getExpectedStorageLocation(); + return storages; } /** Get the number of expected locations */ @@ -279,27 +278,29 @@ public class BlockInfoUnderConstruction extends BlockInfo { if (!(replicas.get(i).isAlive() && !replicas.get(i).getChosenAsPrimary())) { continue; } - if (replicas.get(i).getExpectedLocation().getLastUpdate() > mostRecentLastUpdate) { - primary = replicas.get(i); + final ReplicaUnderConstruction ruc = replicas.get(i); + final long lastUpdate = ruc.getExpectedStorageLocation().getDatanodeDescriptor().getLastUpdate(); + if (lastUpdate > mostRecentLastUpdate) { primaryNodeIndex = i; - mostRecentLastUpdate = primary.getExpectedLocation().getLastUpdate(); + primary = ruc; + mostRecentLastUpdate = lastUpdate; } } if (primary != null) { - primary.getExpectedLocation().addBlockToBeRecovered(this); + primary.getExpectedStorageLocation().getDatanodeDescriptor().addBlockToBeRecovered(this); primary.setChosenAsPrimary(true); NameNode.blockStateChangeLog.info("BLOCK* " + this + " recovery started, primary=" + primary); } } - void addReplicaIfNotPresent(DatanodeDescriptor dn, + void addReplicaIfNotPresent(DatanodeStorageInfo storage, Block block, ReplicaState rState) { for(ReplicaUnderConstruction r : replicas) - if(r.getExpectedLocation() == dn) + if(r.getExpectedStorageLocation() == storage) return; - replicas.add(new ReplicaUnderConstruction(block, dn, rState)); + replicas.add(new ReplicaUnderConstruction(block, storage, rState)); } @Override // BlockInfo diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 77d75a4eb57..f26f0e953a1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -500,9 +500,8 @@ public class BlockManager { Collection corruptNodes = corruptReplicas.getNodes(block); - for (Iterator jt = blocksMap.nodeIterator(block); - jt.hasNext();) { - DatanodeDescriptor node = jt.next(); + for (DatanodeStorageInfo storage : blocksMap.getStorages(block)) { + final DatanodeDescriptor node = storage.getDatanodeDescriptor(); String state = ""; if (corruptNodes != null && corruptNodes.contains(node)) { state = "(corrupt)"; @@ -662,10 +661,9 @@ public class BlockManager { assert oldBlock == getStoredBlock(oldBlock) : "last block of the file is not in blocksMap"; - DatanodeDescriptor[] targets = getNodes(oldBlock); + DatanodeStorageInfo[] targets = getStorages(oldBlock); - BlockInfoUnderConstruction ucBlock = - bc.setLastBlock(oldBlock, targets); + BlockInfoUnderConstruction ucBlock = bc.setLastBlock(oldBlock, targets); blocksMap.replaceBlock(ucBlock); // Remove block from replication queue. @@ -675,9 +673,8 @@ public class BlockManager { pendingReplications.remove(ucBlock); // remove this block from the list of pending blocks to be deleted. - for (DatanodeDescriptor dd : targets) { - String datanodeId = dd.getStorageID(); - invalidateBlocks.remove(datanodeId, oldBlock); + for (DatanodeStorageInfo storage : targets) { + invalidateBlocks.remove(storage.getStorageID(), oldBlock); } // Adjust safe-mode totals, since under-construction blocks don't @@ -699,9 +696,8 @@ public class BlockManager { private List getValidLocations(Block block) { ArrayList machineSet = new ArrayList(blocksMap.numNodes(block)); - for(Iterator it = - blocksMap.nodeIterator(block); it.hasNext();) { - String storageID = it.next().getStorageID(); + for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) { + final String storageID = storage.getStorageID(); // filter invalidate replicas if(!invalidateBlocks.contains(storageID, block)) { machineSet.add(storageID); @@ -775,9 +771,9 @@ public class BlockManager { + ", blk=" + blk); } final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)blk; - final DatanodeDescriptor[] locations = uc.getExpectedLocations(); + final DatanodeStorageInfo[] storages = uc.getExpectedStorageLocations(); final ExtendedBlock eb = new ExtendedBlock(namesystem.getBlockPoolId(), blk); - return new LocatedBlock(eb, locations, pos, false); + return LocatedBlock.createLocatedBlock(eb, storages, pos, false); } // get block locations @@ -795,9 +791,8 @@ public class BlockManager { final DatanodeDescriptor[] machines = new DatanodeDescriptor[numMachines]; int j = 0; if (numMachines > 0) { - for(Iterator it = blocksMap.nodeIterator(blk); - it.hasNext();) { - final DatanodeDescriptor d = it.next(); + for(DatanodeStorageInfo storage : blocksMap.getStorages(blk)) { + final DatanodeDescriptor d = storage.getDatanodeDescriptor(); final boolean replicaCorrupt = corruptReplicas.isReplicaCorrupt(blk, d); if (isCorrupt || (!isCorrupt && !replicaCorrupt)) machines[j++] = d; @@ -1017,9 +1012,8 @@ public class BlockManager { */ private void addToInvalidates(Block b) { StringBuilder datanodes = new StringBuilder(); - for (Iterator it = blocksMap.nodeIterator(b); it - .hasNext();) { - DatanodeDescriptor node = it.next(); + for(DatanodeStorageInfo storage : blocksMap.getStorages(b)) { + final DatanodeDescriptor node = storage.getDatanodeDescriptor(); invalidateBlocks.add(b, node, false); datanodes.append(node).append(" "); } @@ -1466,10 +1460,10 @@ public class BlockManager { int decommissioned = 0; int corrupt = 0; int excess = 0; - Iterator it = blocksMap.nodeIterator(block); + Collection nodesCorrupt = corruptReplicas.getNodes(block); - while(it.hasNext()) { - DatanodeDescriptor node = it.next(); + for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) { + final DatanodeDescriptor node = storage.getDatanodeDescriptor(); LightWeightLinkedSet excessBlocks = excessReplicateMap.get(node.getStorageID()); if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) @@ -1790,7 +1784,7 @@ public class BlockManager { // If block is under construction, add this replica to its list if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) { ((BlockInfoUnderConstruction)storedBlock).addReplicaIfNotPresent( - node, iblk, reportedState); + node.getStorageInfo(storageID), iblk, reportedState); //and fall through to next clause } //add replica if appropriate @@ -2093,7 +2087,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block DatanodeDescriptor node, String storageID, ReplicaState reportedState) throws IOException { - block.addReplicaIfNotPresent(node, block, reportedState); + block.addReplicaIfNotPresent(node.getStorageInfo(storageID), block, reportedState); if (reportedState == ReplicaState.FINALIZED && block.findDatanode(node) < 0) { addStoredBlock(block, node, storageID, null, true); } @@ -2425,9 +2419,8 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block Collection nonExcess = new ArrayList(); Collection corruptNodes = corruptReplicas .getNodes(block); - for (Iterator it = blocksMap.nodeIterator(block); - it.hasNext();) { - DatanodeDescriptor cur = it.next(); + for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) { + final DatanodeDescriptor cur = storage.getDatanodeDescriptor(); if (cur.areBlockContentsStale()) { LOG.info("BLOCK* processOverReplicatedBlock: " + "Postponing processing of over-replicated " + @@ -2747,10 +2740,9 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block int corrupt = 0; int excess = 0; int stale = 0; - Iterator nodeIter = blocksMap.nodeIterator(b); Collection nodesCorrupt = corruptReplicas.getNodes(b); - while (nodeIter.hasNext()) { - DatanodeDescriptor node = nodeIter.next(); + for(DatanodeStorageInfo storage : blocksMap.getStorages(b)) { + final DatanodeDescriptor node = storage.getDatanodeDescriptor(); if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) { corrupt++; } else if (node.isDecommissionInProgress() || node.isDecommissioned()) { @@ -2787,10 +2779,9 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block } // else proceed with fast case int live = 0; - Iterator nodeIter = blocksMap.nodeIterator(b); Collection nodesCorrupt = corruptReplicas.getNodes(b); - while (nodeIter.hasNext()) { - DatanodeDescriptor node = nodeIter.next(); + for(DatanodeStorageInfo storage : blocksMap.getStorages(b)) { + final DatanodeDescriptor node = storage.getDatanodeDescriptor(); if ((nodesCorrupt == null) || (!nodesCorrupt.contains(node))) live++; } @@ -2802,10 +2793,9 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block int curReplicas = num.liveReplicas(); int curExpectedReplicas = getReplication(block); BlockCollection bc = blocksMap.getBlockCollection(block); - Iterator nodeIter = blocksMap.nodeIterator(block); StringBuilder nodeList = new StringBuilder(); - while (nodeIter.hasNext()) { - DatanodeDescriptor node = nodeIter.next(); + for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) { + final DatanodeDescriptor node = storage.getDatanodeDescriptor(); nodeList.append(node); nodeList.append(" "); } @@ -2902,14 +2892,13 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block return blocksMap.size(); } - public DatanodeDescriptor[] getNodes(BlockInfo block) { - DatanodeDescriptor[] nodes = - new DatanodeDescriptor[block.numNodes()]; - Iterator it = blocksMap.nodeIterator(block); - for (int i = 0; it != null && it.hasNext(); i++) { - nodes[i] = it.next(); + public DatanodeStorageInfo[] getStorages(BlockInfo block) { + final DatanodeStorageInfo[] storages = new DatanodeStorageInfo[block.numNodes()]; + int i = 0; + for(DatanodeStorageInfo s : blocksMap.getStorages(block)) { + storages[i++] = s; } - return nodes; + return storages; } public int getTotalBlocks() { @@ -3038,9 +3027,8 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block corruptReplicas.getNodes(b); int numExpectedReplicas = getReplication(b); String rackName = null; - for (Iterator it = blocksMap.nodeIterator(b); - it.hasNext();) { - DatanodeDescriptor cur = it.next(); + for(DatanodeStorageInfo storage : blocksMap.getStorages(b)) { + final DatanodeDescriptor cur = storage.getDatanodeDescriptor(); if (!cur.isDecommissionInProgress() && !cur.isDecommissioned()) { if ((corruptNodes == null ) || !corruptNodes.contains(cur)) { if (numExpectedReplicas == 1 || @@ -3084,8 +3072,8 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block } /** @return an iterator of the datanodes. */ - public Iterator datanodeIterator(final Block block) { - return blocksMap.nodeIterator(block); + public Iterable getStorages(final Block block) { + return blocksMap.getStorages(block); } public int numCorruptReplicas(Block block) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java index 1e454c9bc86..8e9f706f6fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java @@ -29,11 +29,11 @@ import org.apache.hadoop.util.LightWeightGSet; * the datanodes that store the block. */ class BlocksMap { - private static class NodeIterator implements Iterator { + private static class StorageIterator implements Iterator { private BlockInfo blockInfo; private int nextIdx = 0; - NodeIterator(BlockInfo blkInfo) { + StorageIterator(BlockInfo blkInfo) { this.blockInfo = blkInfo; } @@ -44,8 +44,8 @@ class BlocksMap { } @Override - public DatanodeDescriptor next() { - return blockInfo.getDatanode(nextIdx++); + public DatanodeStorageInfo next() { + return blockInfo.getStorageInfo(nextIdx++); } @Override @@ -115,18 +115,23 @@ class BlocksMap { /** * Searches for the block in the BlocksMap and - * returns Iterator that iterates through the nodes the block belongs to. + * returns {@link Iterable} that iterates through the nodes the block belongs to. */ - Iterator nodeIterator(Block b) { - return nodeIterator(blocks.get(b)); + Iterable getStorages(Block b) { + return getStorages(blocks.get(b)); } /** * For a block that has already been retrieved from the BlocksMap - * returns Iterator that iterates through the nodes the block belongs to. + * returns {@link Iterable} that iterates through the nodes the block belongs to. */ - Iterator nodeIterator(BlockInfo storedBlock) { - return new NodeIterator(storedBlock); + Iterable getStorages(final BlockInfo storedBlock) { + return new Iterable() { + @Override + public Iterator iterator() { + return new StorageIterator(storedBlock); + } + }; } /** counts number of containing nodes. Better than using iterator. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index e9f4fdb636b..62235a7bba1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -23,7 +23,13 @@ import java.io.IOException; import java.io.PrintWriter; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.NavigableMap; +import java.util.TreeMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -48,10 +54,17 @@ import org.apache.hadoop.hdfs.server.namenode.HostFileManager.EntrySet; import org.apache.hadoop.hdfs.server.namenode.HostFileManager.MutableEntrySet; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; -import org.apache.hadoop.hdfs.server.protocol.*; +import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand; +import org.apache.hadoop.hdfs.server.protocol.BlockCommand; +import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; +import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; +import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException; +import org.apache.hadoop.hdfs.server.protocol.RegisterCommand; import org.apache.hadoop.hdfs.util.CyclicIteration; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.CachedDNSToSwitchMapping; import org.apache.hadoop.net.DNSToSwitchMapping; @@ -423,6 +436,20 @@ public class DatanodeManager { return node; } + public DatanodeStorageInfo[] getDatanodeStorageInfos( + DatanodeID[] datanodeID, String[] storageIDs) + throws UnregisteredNodeException { + if (datanodeID.length == 0) { + return null; + } + final DatanodeStorageInfo[] storages = new DatanodeStorageInfo[datanodeID.length]; + for(int i = 0; i < datanodeID.length; i++) { + final DatanodeDescriptor dd = getDatanode(datanodeID[i]); + storages[i] = dd.getStorageInfo(storageIDs[i]); + } + return storages; + } + /** Prints information about all datanodes. */ void datanodeDump(final PrintWriter out) { synchronized (datanodeMap) { @@ -1151,32 +1178,32 @@ public class DatanodeManager { BlockRecoveryCommand brCommand = new BlockRecoveryCommand( blocks.length); for (BlockInfoUnderConstruction b : blocks) { - DatanodeDescriptor[] expectedLocations = b.getExpectedLocations(); + final DatanodeStorageInfo[] storages = b.getExpectedStorageLocations(); // Skip stale nodes during recovery - not heart beated for some time (30s by default). - List recoveryLocations = - new ArrayList(expectedLocations.length); - for (int i = 0; i < expectedLocations.length; i++) { - if (!expectedLocations[i].isStale(this.staleInterval)) { - recoveryLocations.add(expectedLocations[i]); + final List recoveryLocations = + new ArrayList(storages.length); + for (int i = 0; i < storages.length; i++) { + if (!storages[i].getDatanodeDescriptor().isStale(staleInterval)) { + recoveryLocations.add(storages[i]); } } // If we only get 1 replica after eliminating stale nodes, then choose all // replicas for recovery and let the primary data node handle failures. if (recoveryLocations.size() > 1) { - if (recoveryLocations.size() != expectedLocations.length) { + if (recoveryLocations.size() != storages.length) { LOG.info("Skipped stale nodes for recovery : " + - (expectedLocations.length - recoveryLocations.size())); + (storages.length - recoveryLocations.size())); } brCommand.add(new RecoveringBlock( new ExtendedBlock(blockPoolId, b), - recoveryLocations.toArray(new DatanodeDescriptor[recoveryLocations.size()]), + DatanodeStorageInfo.toDatanodeInfos(recoveryLocations), b.getBlockRecoveryId())); } else { // If too many replicas are stale, then choose all replicas to participate // in block recovery. brCommand.add(new RecoveringBlock( new ExtendedBlock(blockPoolId, b), - expectedLocations, + DatanodeStorageInfo.toDatanodeInfos(storages), b.getBlockRecoveryId())); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index 805e334acbf..53fcde1c691 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -17,9 +17,12 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import java.util.Arrays; import java.util.Iterator; +import java.util.List; import org.apache.hadoop.hdfs.StorageType; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State; import org.apache.hadoop.hdfs.server.protocol.StorageReport; @@ -29,6 +32,17 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport; * by this class. */ public class DatanodeStorageInfo { + static DatanodeInfo[] toDatanodeInfos(DatanodeStorageInfo[] storages) { + return toDatanodeInfos(Arrays.asList(storages)); + } + static DatanodeInfo[] toDatanodeInfos(List storages) { + final DatanodeInfo[] datanodes = new DatanodeInfo[storages.size()]; + for(int i = 0; i < storages.size(); i++) { + datanodes[i] = storages.get(i).getDatanodeDescriptor(); + } + return datanodes; + } + /** * Iterates over the list of blocks belonging to the data-node. */ @@ -65,7 +79,7 @@ public class DatanodeStorageInfo { private long remaining; private volatile BlockInfo blockList = null; - DatanodeStorageInfo(DatanodeDescriptor dn, DatanodeStorage s) { + public DatanodeStorageInfo(DatanodeDescriptor dn, DatanodeStorage s) { this.dn = dn; this.storageID = s.getStorageID(); this.storageType = s.getStorageType(); @@ -92,6 +106,10 @@ public class DatanodeStorageInfo { return storageID; } + public StorageType getStorageType() { + return storageType; + } + public long getCapacity() { return capacity; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/MutableBlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/MutableBlockCollection.java index 41975d3371d..5d63fefaaf5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/MutableBlockCollection.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/MutableBlockCollection.java @@ -34,5 +34,5 @@ public interface MutableBlockCollection extends BlockCollection { * and set the locations. */ public BlockInfoUnderConstruction setLastBlock(BlockInfo lastBlock, - DatanodeDescriptor[] locations) throws IOException; + DatanodeStorageInfo[] storages) throws IOException; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 51642a8b23a..e52560b6b73 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; @@ -345,7 +346,7 @@ public class FSDirectory implements Closeable { * Add a block to the file. Returns a reference to the added block. */ BlockInfo addBlock(String path, INodesInPath inodesInPath, Block block, - DatanodeDescriptor targets[]) throws IOException { + DatanodeStorageInfo[] targets) throws IOException { waitForReady(); writeLock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index d83d53a5f35..94ae9d8cc72 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -168,6 +168,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStatistics; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.blockmanagement.OutOfV1GenerationStampsException; import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; @@ -2484,8 +2485,14 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } // choose targets for the new block to be allocated. - final DatanodeDescriptor targets[] = getBlockManager().chooseTarget( + // TODO: chooseTarget(..) should be changed to return DatanodeStorageInfo's + final DatanodeDescriptor chosenDatanodes[] = getBlockManager().chooseTarget( src, replication, clientNode, excludedNodes, blockSize, favoredNodes); + final DatanodeStorageInfo[] targets = new DatanodeStorageInfo[chosenDatanodes.length]; + for(int i = 0; i < targets.length; i++) { + final DatanodeDescriptor dd = chosenDatanodes[i]; + targets[i] = dd.getStorageInfos().iterator().next(); + } // Part II. // Allocate a new block, add it to the INode and the BlocksMap. @@ -2607,7 +2614,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, src + ". Returning previously allocated block " + lastBlockInFile); long offset = pendingFile.computeFileSize(); onRetryBlock[0] = makeLocatedBlock(lastBlockInFile, - ((BlockInfoUnderConstruction)lastBlockInFile).getExpectedLocations(), + ((BlockInfoUnderConstruction)lastBlockInFile).getExpectedStorageLocations(), offset); return iip; } else { @@ -2625,11 +2632,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats, return iip; } - LocatedBlock makeLocatedBlock(Block blk, - DatanodeInfo[] locs, + LocatedBlock makeLocatedBlock(Block blk, DatanodeStorageInfo[] locs, long offset) throws IOException { - LocatedBlock lBlk = new LocatedBlock( - getExtendedBlock(blk), locs, offset); + LocatedBlock lBlk = LocatedBlock.createLocatedBlock( + getExtendedBlock(blk), locs, offset, false); getBlockManager().setBlockToken( lBlk, BlockTokenSecretManager.AccessMode.WRITE); return lBlk; @@ -2852,13 +2858,14 @@ public class FSNamesystem implements Namesystem, FSClusterStats, * @throws QuotaExceededException If addition of block exceeds space quota */ BlockInfo saveAllocatedBlock(String src, INodesInPath inodesInPath, - Block newBlock, DatanodeDescriptor targets[]) throws IOException { + Block newBlock, DatanodeStorageInfo[] targets) + throws IOException { assert hasWriteLock(); BlockInfo b = dir.addBlock(src, inodesInPath, newBlock, targets); NameNode.stateChangeLog.info("BLOCK* allocateBlock: " + src + ". " + getBlockPoolId() + " " + b); - for (DatanodeDescriptor dn : targets) { - dn.incBlocksScheduled(); + for (DatanodeStorageInfo storage : targets) { + storage.getDatanodeDescriptor().incBlocksScheduled(); } return b; } @@ -3622,7 +3629,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)lastBlock; // setup the last block locations from the blockManager if not known if (uc.getNumExpectedLocations() == 0) { - uc.setExpectedLocations(blockManager.getNodes(lastBlock)); + uc.setExpectedLocations(blockManager.getStorages(lastBlock)); } // start recovery of the last block for this file long blockRecoveryId = nextGenerationStamp(isLegacyBlock(uc)); @@ -3777,24 +3784,18 @@ public class FSNamesystem implements Namesystem, FSClusterStats, // find the DatanodeDescriptor objects // There should be no locations in the blockManager till now because the // file is underConstruction - DatanodeDescriptor[] descriptors = null; - if (newtargets.length > 0) { - descriptors = new DatanodeDescriptor[newtargets.length]; - for(int i = 0; i < newtargets.length; i++) { - descriptors[i] = blockManager.getDatanodeManager().getDatanode( - newtargets[i]); - } - } - if ((closeFile) && (descriptors != null)) { + final DatanodeStorageInfo[] storages = blockManager.getDatanodeManager() + .getDatanodeStorageInfos(newtargets, newtargetstorages); + if (closeFile && storages != null) { // the file is getting closed. Insert block locations into blockManager. // Otherwise fsck will report these blocks as MISSING, especially if the // blocksReceived from Datanodes take a long time to arrive. - for (int i = 0; i < descriptors.length; i++) { - descriptors[i].addBlock(newtargetstorages[i], storedBlock); + for (int i = 0; i < storages.length; i++) { + storages[i].addBlock(storedBlock); } } // add pipeline locations into the INodeUnderConstruction - pendingFile.setLastBlock(storedBlock, descriptors); + pendingFile.setLastBlock(storedBlock, storages); } if (closeFile) { @@ -5639,7 +5640,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, * @throws IOException if any error occurs */ void updatePipeline(String clientName, ExtendedBlock oldBlock, - ExtendedBlock newBlock, DatanodeID[] newNodes) + ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs) throws IOException { CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); if (cacheEntry != null && cacheEntry.isSuccess()) { @@ -5662,7 +5663,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, assert newBlock.getBlockId()==oldBlock.getBlockId() : newBlock + " and " + oldBlock + " has different block identifier"; updatePipelineInternal(clientName, oldBlock, newBlock, newNodes, - cacheEntry != null); + newStorageIDs, cacheEntry != null); success = true; } finally { writeUnlock(); @@ -5674,7 +5675,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, /** @see #updatePipeline(String, ExtendedBlock, ExtendedBlock, DatanodeID[]) */ private void updatePipelineInternal(String clientName, ExtendedBlock oldBlock, - ExtendedBlock newBlock, DatanodeID[] newNodes, boolean logRetryCache) + ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs, + boolean logRetryCache) throws IOException { assert hasWriteLock(); // check the vadility of the block and lease holder name @@ -5698,15 +5700,9 @@ public class FSNamesystem implements Namesystem, FSClusterStats, blockinfo.setNumBytes(newBlock.getNumBytes()); // find the DatanodeDescriptor objects - final DatanodeManager dm = getBlockManager().getDatanodeManager(); - DatanodeDescriptor[] descriptors = null; - if (newNodes.length > 0) { - descriptors = new DatanodeDescriptor[newNodes.length]; - for(int i = 0; i < newNodes.length; i++) { - descriptors[i] = dm.getDatanode(newNodes[i]); - } - } - blockinfo.setExpectedLocations(descriptors); + final DatanodeStorageInfo[] storages = blockManager.getDatanodeManager() + .getDatanodeStorageInfos(newNodes, newStorageIDs); + blockinfo.setExpectedLocations(storages); String src = leaseManager.findPath(pendingFile); dir.persistBlocks(src, pendingFile, logRetryCache); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java index 9dd38ff320b..2f9fbb0af7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.blockmanagement.MutableBlockCollection; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileUnderConstructionWithSnapshot; @@ -180,7 +181,7 @@ public class INodeFileUnderConstruction extends INodeFile implements MutableBloc */ @Override public BlockInfoUnderConstruction setLastBlock(BlockInfo lastBlock, - DatanodeDescriptor[] targets) throws IOException { + DatanodeStorageInfo[] targets) throws IOException { if (numBlocks() == 0) { throw new IOException("Failed to set last block: File is empty."); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 62a17a96f5b..defe743898f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -611,9 +611,9 @@ class NameNodeRpcServer implements NamenodeProtocols { @Override // ClientProtocol public void updatePipeline(String clientName, ExtendedBlock oldBlock, - ExtendedBlock newBlock, DatanodeID[] newNodes) + ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs) throws IOException { - namesystem.updatePipeline(clientName, oldBlock, newBlock, newNodes); + namesystem.updatePipeline(clientName, oldBlock, newBlock, newNodes, newStorageIDs); } @Override // DatanodeProtocol diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java index 1989784ba53..aabc073c899 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; @@ -1065,11 +1066,10 @@ class NamenodeJspHelper { } doc.startTag("replicas"); - for(final Iterator it = blockManager.datanodeIterator(block); - it.hasNext(); ) { + for(DatanodeStorageInfo storage : blockManager.getStorages(block)) { doc.startTag("replica"); - DatanodeDescriptor dd = it.next(); + DatanodeDescriptor dd = storage.getDatanodeDescriptor(); doc.startTag("host_name"); doc.pcdata(dd.getHostName()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index 95fcc50ebd3..3f09aeeb3c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -437,6 +437,7 @@ message UpdatePipelineRequestProto { required ExtendedBlockProto oldBlock = 2; required ExtendedBlockProto newBlock = 3; repeated DatanodeIDProto newNodes = 4; + repeated string storageIDs = 5; } message UpdatePipelineResponseProto { // void response diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 9709e888480..da0b8322aa6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -134,6 +134,7 @@ message LocatedBlockProto { required hadoop.common.TokenProto blockToken = 5; repeated StorageTypeProto storageTypes = 6; + repeated string storageIDs = 7; } message DataEncryptionKeyProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index f0c10b0a2fe..696a320e23c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -61,8 +61,8 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem.Statistics; import org.apache.hadoop.fs.Options.Rename; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster.NameNodeInfo; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.protocol.DatanodeID; @@ -79,6 +79,7 @@ import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.datanode.DataNode; @@ -86,6 +87,7 @@ import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; @@ -856,6 +858,13 @@ public class DFSTestUtil { rackLocation); } + public static DatanodeStorageInfo createDatanodeStorageInfo( + String storageID, String ip) { + return new DatanodeStorageInfo( + getDatanodeDescriptor(ip, "defaultRack"), + new DatanodeStorage(storageID)); + } + public static DatanodeDescriptor getDatanodeDescriptor(String ipAddr, int port, String rackLocation) { DatanodeID dnId = new DatanodeID(ipAddr, "host", "", port, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java index 006ee42bb58..2f659417479 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java @@ -85,9 +85,8 @@ public class BlockManagerTestUtil { final Set rackSet = new HashSet(0); final Collection corruptNodes = getCorruptReplicas(blockManager).getNodes(b); - for (Iterator it = blockManager.blocksMap.nodeIterator(b); - it.hasNext();) { - DatanodeDescriptor cur = it.next(); + for(DatanodeStorageInfo storage : blockManager.blocksMap.getStorages(b)) { + final DatanodeDescriptor cur = storage.getDatanodeDescriptor(); if (!cur.isDecommissionInProgress() && !cur.isDecommissioned()) { if ((corruptNodes == null ) || !corruptNodes.contains(cur)) { String rackName = cur.getNetworkLocation(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java index cafc8227147..703d344347a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java @@ -31,18 +31,19 @@ import org.junit.Test; public class TestBlockInfoUnderConstruction { @Test public void testInitializeBlockRecovery() throws Exception { - DatanodeDescriptor dd1 = DFSTestUtil.getDatanodeDescriptor("10.10.1.1", - "default"); - DatanodeDescriptor dd2 = DFSTestUtil.getDatanodeDescriptor("10.10.1.2", - "default"); - DatanodeDescriptor dd3 = DFSTestUtil.getDatanodeDescriptor("10.10.1.3", - "default"); + DatanodeStorageInfo s1 = DFSTestUtil.createDatanodeStorageInfo("10.10.1.1", "s1"); + DatanodeDescriptor dd1 = s1.getDatanodeDescriptor(); + DatanodeStorageInfo s2 = DFSTestUtil.createDatanodeStorageInfo("10.10.1.2", "s2"); + DatanodeDescriptor dd2 = s2.getDatanodeDescriptor(); + DatanodeStorageInfo s3 = DFSTestUtil.createDatanodeStorageInfo("10.10.1.3", "s3"); + DatanodeDescriptor dd3 = s3.getDatanodeDescriptor(); + dd1.isAlive = dd2.isAlive = dd3.isAlive = true; BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction( new Block(0, 0, GenerationStamp.LAST_RESERVED_STAMP), 3, BlockUCState.UNDER_CONSTRUCTION, - new DatanodeDescriptor[] {dd1, dd2, dd3}); + new DatanodeStorageInfo[] {s1, s2, s3}); // Recovery attempt #1. long currentTime = System.currentTimeMillis(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java index bbb83070a30..fc5ce60b4d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import static org.junit.Assert.assertEquals; import java.util.ArrayList; -import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -162,10 +161,13 @@ public class TestHeartbeatHandling { dd1.setLastUpdate(System.currentTimeMillis()); dd2.setLastUpdate(System.currentTimeMillis()); dd3.setLastUpdate(System.currentTimeMillis()); + final DatanodeStorageInfo[] storages = { + dd1.getStorageInfos().iterator().next(), + dd2.getStorageInfos().iterator().next(), + dd3.getStorageInfos().iterator().next()}; BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction( new Block(0, 0, GenerationStamp.LAST_RESERVED_STAMP), 3, - BlockUCState.UNDER_RECOVERY, - new DatanodeDescriptor[] {dd1, dd2, dd3}); + BlockUCState.UNDER_RECOVERY, storages); dd1.addBlockToBeRecovered(blockInfo); DatanodeCommand[] cmds = NameNodeAdapter.sendHeartBeat(nodeReg1, dd1, namesystem).getCommands(); @@ -187,8 +189,7 @@ public class TestHeartbeatHandling { dd3.setLastUpdate(System.currentTimeMillis()); blockInfo = new BlockInfoUnderConstruction( new Block(0, 0, GenerationStamp.LAST_RESERVED_STAMP), 3, - BlockUCState.UNDER_RECOVERY, - new DatanodeDescriptor[] {dd1, dd2, dd3}); + BlockUCState.UNDER_RECOVERY, storages); dd1.addBlockToBeRecovered(blockInfo); cmds = NameNodeAdapter.sendHeartBeat(nodeReg1, dd1, namesystem).getCommands(); assertEquals(1, cmds.length); @@ -209,8 +210,7 @@ public class TestHeartbeatHandling { dd3.setLastUpdate(System.currentTimeMillis() - 80 * 1000); blockInfo = new BlockInfoUnderConstruction( new Block(0, 0, GenerationStamp.LAST_RESERVED_STAMP), 3, - BlockUCState.UNDER_RECOVERY, - new DatanodeDescriptor[] {dd1, dd2, dd3}); + BlockUCState.UNDER_RECOVERY, storages); dd1.addBlockToBeRecovered(blockInfo); cmds = NameNodeAdapter.sendHeartBeat(nodeReg1, dd1, namesystem).getCommands(); assertEquals(1, cmds.length); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java index 86994bed3ac..d66c9254e5c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java @@ -98,11 +98,9 @@ public class TestNodeCount { } // find out a non-excess node - final Iterator iter = bm.blocksMap - .nodeIterator(block.getLocalBlock()); DatanodeDescriptor nonExcessDN = null; - while (iter.hasNext()) { - DatanodeDescriptor dn = iter.next(); + for(DatanodeStorageInfo storage : bm.blocksMap.getStorages(block.getLocalBlock())) { + final DatanodeDescriptor dn = storage.getDatanodeDescriptor(); Collection blocks = bm.excessReplicateMap.get(dn.getStorageID()); if (blocks == null || !blocks.contains(block.getLocalBlock()) ) { nonExcessDN = dn; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index 547e8536f2b..ee86e9dac44 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.namenode.FSClusterStats; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.util.Time; @@ -71,6 +72,7 @@ public class TestReplicationPolicy { private static BlockPlacementPolicy replicator; private static final String filename = "/dummyfile.txt"; private static DatanodeDescriptor dataNodes[]; + private static String[] storageIDs; // The interval for marking a datanode as stale, private static long staleInterval = DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_DEFAULT; @@ -1143,11 +1145,12 @@ public class TestReplicationPolicy { info.setBlockCollection(mbc); bm.addBlockCollection(info, mbc); - DatanodeDescriptor[] dnAry = {dataNodes[0]}; + DatanodeStorageInfo[] storageAry = {new DatanodeStorageInfo( + dataNodes[0], new DatanodeStorage("s1"))}; final BlockInfoUnderConstruction ucBlock = info.convertToBlockUnderConstruction(BlockUCState.UNDER_CONSTRUCTION, - dnAry); - when(mbc.setLastBlock((BlockInfo) any(), (DatanodeDescriptor[]) any())) + storageAry); + when(mbc.setLastBlock((BlockInfo) any(), (DatanodeStorageInfo[]) any())) .thenReturn(ucBlock); bm.convertLastBlockToUnderConstruction(mbc); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java index 7decc4b8978..210f0e90421 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlocks.java @@ -48,7 +48,8 @@ public class TestUnderReplicatedBlocks { // but the block does not get put into the under-replicated blocks queue final BlockManager bm = cluster.getNamesystem().getBlockManager(); ExtendedBlock b = DFSTestUtil.getFirstBlock(fs, FILE_PATH); - DatanodeDescriptor dn = bm.blocksMap.nodeIterator(b.getLocalBlock()).next(); + DatanodeDescriptor dn = bm.blocksMap.getStorages(b.getLocalBlock()) + .iterator().next().getDatanodeDescriptor(); bm.addToInvalidates(b.getLocalBlock(), dn); // Compute the invalidate work in NN, and trigger the heartbeat from DN BlockManagerTestUtil.computeAllPendingWork(bm); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java index 53007196200..23bd4994a37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.junit.Test; @@ -50,7 +50,7 @@ public class TestCommitBlockSynchronization { throws IOException { Configuration conf = new Configuration(); FSImage image = new FSImage(conf); - DatanodeDescriptor[] targets = new DatanodeDescriptor[0]; + final DatanodeStorageInfo[] targets = {}; FSNamesystem namesystem = new FSNamesystem(conf, image); FSNamesystem namesystemSpy = spy(namesystem); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java index 3c8b91be8d2..7c87ab0d664 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -523,16 +524,17 @@ public class TestPipelinesFailover { (BlockInfoUnderConstruction)storedBlock; // We expect that the replica with the most recent heart beat will be // the one to be in charge of the synchronization / recovery protocol. - DatanodeDescriptor[] datanodes = ucBlock.getExpectedLocations(); - DatanodeDescriptor expectedPrimary = datanodes[0]; - long mostRecentLastUpdate = expectedPrimary.getLastUpdate(); - for (int i = 1; i < datanodes.length; i++) { - if (datanodes[i].getLastUpdate() > mostRecentLastUpdate) { - expectedPrimary = datanodes[i]; - mostRecentLastUpdate = expectedPrimary.getLastUpdate(); + final DatanodeStorageInfo[] storages = ucBlock.getExpectedStorageLocations(); + DatanodeStorageInfo expectedPrimary = storages[0]; + long mostRecentLastUpdate = expectedPrimary.getDatanodeDescriptor().getLastUpdate(); + for (int i = 1; i < storages.length; i++) { + final long lastUpdate = storages[i].getDatanodeDescriptor().getLastUpdate(); + if (lastUpdate > mostRecentLastUpdate) { + expectedPrimary = storages[i]; + mostRecentLastUpdate = lastUpdate; } } - return expectedPrimary; + return expectedPrimary.getDatanodeDescriptor(); } private DistributedFileSystem createFsAsOtherUser( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java index dff44a0c89c..e907f7669af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java @@ -702,9 +702,10 @@ public class TestRetryCacheWithHA { DatanodeInfo[] newNodes = new DatanodeInfo[2]; newNodes[0] = nodes[0]; newNodes[1] = nodes[1]; + String[] storageIDs = {"s0", "s1"}; client.getNamenode().updatePipeline(client.getClientName(), oldBlock, - newBlock, newNodes); + newBlock, newNodes, storageIDs); out.close(); } @@ -714,10 +715,10 @@ public class TestRetryCacheWithHA { .getNamesystem(0).getFSDirectory().getINode4Write(file).asFile(); BlockInfoUnderConstruction blkUC = (BlockInfoUnderConstruction) (fileNode.getBlocks())[1]; - int datanodeNum = blkUC.getExpectedLocations().length; + int datanodeNum = blkUC.getExpectedStorageLocations().length; for (int i = 0; i < CHECKTIMES && datanodeNum != 2; i++) { Thread.sleep(1000); - datanodeNum = blkUC.getExpectedLocations().length; + datanodeNum = blkUC.getExpectedStorageLocations().length; } return datanodeNum == 2; } From 282be1b38e5cd141ed7e2b2194bfb67a7c2f7f15 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Sun, 8 Sep 2013 22:53:36 +0000 Subject: [PATCH 10/76] HDFS-5134. Move blockContentsStale, heartbeatedSinceFailover and firstBlockReport from DatanodeDescriptor to DatanodeStorageInfo; and fix a synchronization problem in DatanodeStorageInfo. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1520938 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 4 + .../server/blockmanagement/BlockInfo.java | 8 -- .../server/blockmanagement/BlockManager.java | 32 ++++--- .../blockmanagement/DatanodeDescriptor.java | 83 ++++++------------- .../blockmanagement/DatanodeManager.java | 10 ++- .../blockmanagement/DatanodeStorageInfo.java | 62 +++++++++++++- .../server/namenode/NameNodeRpcServer.java | 3 - .../blockmanagement/BlockManagerTestUtil.java | 1 - .../blockmanagement/TestBlockManager.java | 18 ++-- 9 files changed, 124 insertions(+), 97 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 5a6f2c76c4c..49379bc1f9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -17,3 +17,7 @@ IMPROVEMENTS: (Junping Du via szetszwo) HDFS-5009. Include storage information in the LocatedBlock. (szetszwo) + + HDFS-5134. Move blockContentsStale, heartbeatedSinceFailover and + firstBlockReport from DatanodeDescriptor to DatanodeStorageInfo; and + fix a synchronization problem in DatanodeStorageInfo. (szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index 9f7aabd4668..056628888b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -324,14 +324,6 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { return head; } - int listCount(DatanodeStorageInfo storage) { - int count = 0; - for(BlockInfo cur = this; cur != null; - cur = cur.getNext(cur.findStorageInfo(storage))) - count++; - return count; - } - /** * BlockInfo represents a block that is not being constructed. * In order to start modifying the block, the BlockInfo should be converted diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index f26f0e953a1..a8024f412f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -510,7 +510,7 @@ public class BlockManager { state = "(decommissioned)"; } - if (node.areBlockContentsStale()) { + if (storage.areBlockContentsStale()) { state += " (block deletions maybe out of date)"; } out.print(" " + node + state + " : "); @@ -993,7 +993,14 @@ public class BlockManager { // failover, then we may have been holding up on processing // over-replicated blocks because of it. But we can now // process those blocks. - if (node.areBlockContentsStale()) { + boolean stale = false; + for(DatanodeStorageInfo storage : node.getStorageInfos()) { + if (storage.areBlockContentsStale()) { + stale = true; + break; + } + } + if (stale) { rescanPostponedMisreplicatedBlocks(); } } @@ -1616,14 +1623,16 @@ public class BlockManager { // To minimize startup time, we discard any second (or later) block reports // that we receive while still in startup phase. - if (namesystem.isInStartupSafeMode() && !node.isFirstBlockReport()) { + final DatanodeStorageInfo storageInfo = node.getStorageInfo(storage.getStorageID()); + if (namesystem.isInStartupSafeMode() + && storageInfo.getBlockReportCount() > 0) { blockLog.info("BLOCK* processReport: " + "discarded non-initial block report from " + nodeID + " because namenode still in startup phase"); return; } - if (node.numBlocks() == 0) { + if (storageInfo.numBlocks() == 0) { // The first block report can be processed a lot more efficiently than // ordinary block reports. This shortens restart times. processFirstBlockReport(node, storage.getStorageID(), newReport); @@ -1633,9 +1642,9 @@ public class BlockManager { // Now that we have an up-to-date block report, we know that any // deletions from a previous NN iteration have been accounted for. - boolean staleBefore = node.areBlockContentsStale(); - node.receivedBlockReport(); - if (staleBefore && !node.areBlockContentsStale()) { + boolean staleBefore = storageInfo.areBlockContentsStale(); + storageInfo.receivedBlockReport(); + if (staleBefore && !storageInfo.areBlockContentsStale()) { LOG.info("BLOCK* processReport: Received first block report from " + node + " after starting up or becoming active. Its block " + "contents are no longer considered stale"); @@ -1747,7 +1756,7 @@ public class BlockManager { final BlockListAsLongs report) throws IOException { if (report == null) return; assert (namesystem.hasWriteLock()); - assert (node.numBlocks() == 0); + assert (node.getStorageInfo(storageID).numBlocks() == 0); BlockReportIterator itBR = report.getBlockReportIterator(); while(itBR.hasNext()) { @@ -2421,10 +2430,11 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block .getNodes(block); for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) { final DatanodeDescriptor cur = storage.getDatanodeDescriptor(); - if (cur.areBlockContentsStale()) { + if (storage.areBlockContentsStale()) { LOG.info("BLOCK* processOverReplicatedBlock: " + "Postponing processing of over-replicated " + - block + " since datanode " + cur + " does not yet have up-to-date " + + block + " since storage + " + storage + + "datanode " + cur + " does not yet have up-to-date " + "block information."); postponeBlock(block); return; @@ -2756,7 +2766,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block live++; } } - if (node.areBlockContentsStale()) { + if (storage.areBlockContentsStale()) { stale++; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index bc1107beb73..a4ddc3130e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -106,23 +106,6 @@ public class DatanodeDescriptor extends DatanodeInfo { public boolean isAlive = false; public boolean needKeyUpdate = false; - /** - * Set to false on any NN failover, and reset to true - * whenever a block report is received. - */ - private boolean heartbeatedSinceFailover = false; - - /** - * At startup or at any failover, the DNs in the cluster may - * have pending block deletions from a previous incarnation - * of the NameNode. Thus, we consider their block contents - * stale until we have received a block report. When a DN - * is considered stale, any replicas on it are transitively - * considered stale. If any block has at least one stale replica, - * then no invalidations will be processed for this block. - * See HDFS-1972. - */ - private boolean blockContentsStale = true; // A system administrator can tune the balancer bandwidth parameter // (dfs.balance.bandwidthPerSec) dynamically by calling @@ -151,9 +134,6 @@ public class DatanodeDescriptor extends DatanodeInfo { private static final int BLOCKS_SCHEDULED_ROLL_INTERVAL = 600*1000; //10min private int volumeFailures = 0; - /** Set to false after processing first block report */ - private boolean firstBlockReport = true; - /** * When set to true, the node is not in include list and is not allowed * to communicate with the namenode @@ -234,11 +214,15 @@ public class DatanodeDescriptor extends DatanodeInfo { return false; } - DatanodeStorageInfo getStorageInfo(String storageID) { - return storageMap.get(storageID); + public DatanodeStorageInfo getStorageInfo(String storageID) { + synchronized (storageMap) { + return storageMap.get(storageID); + } } public Collection getStorageInfos() { - return storageMap.values(); + synchronized (storageMap) { + return new ArrayList(storageMap.values()); + } } /** @@ -314,9 +298,8 @@ public class DatanodeDescriptor extends DatanodeInfo { } public int numBlocks() { - // TODO: synchronization int blocks = 0; - for (DatanodeStorageInfo entry : storageMap.values()) { + for (DatanodeStorageInfo entry : getStorageInfos()) { blocks += entry.numBlocks(); } return blocks; @@ -334,7 +317,9 @@ public class DatanodeDescriptor extends DatanodeInfo { setXceiverCount(xceiverCount); setLastUpdate(Time.now()); this.volumeFailures = volFailures; - this.heartbeatedSinceFailover = true; + for(DatanodeStorageInfo storage : getStorageInfos()) { + storage.receivedHeartbeat(); + } rollBlocksScheduled(getLastUpdate()); } @@ -380,10 +365,10 @@ public class DatanodeDescriptor extends DatanodeInfo { } Iterator getBlockIterator() { - return new BlockIterator(storageMap.values()); + return new BlockIterator(getStorageInfos()); } Iterator getBlockIterator(final String storageID) { - return new BlockIterator(storageMap.get(storageID)); + return new BlockIterator(getStorageInfo(storageID)); } /** @@ -585,7 +570,11 @@ public class DatanodeDescriptor extends DatanodeInfo { @Override public void updateRegInfo(DatanodeID nodeReg) { super.updateRegInfo(nodeReg); - firstBlockReport = true; // must re-process IBR after re-registration + + // must re-process IBR after re-registration + for(DatanodeStorageInfo storage : getStorageInfos()) { + storage.setBlockReportCount(0); + } } /** @@ -602,26 +591,6 @@ public class DatanodeDescriptor extends DatanodeInfo { this.bandwidth = bandwidth; } - public boolean areBlockContentsStale() { - return blockContentsStale; - } - - public void markStaleAfterFailover() { - heartbeatedSinceFailover = false; - blockContentsStale = true; - } - - public void receivedBlockReport() { - if (heartbeatedSinceFailover) { - blockContentsStale = false; - } - firstBlockReport = false; - } - - boolean isFirstBlockReport() { - return firstBlockReport; - } - @Override public String dumpDatanode() { StringBuilder sb = new StringBuilder(super.dumpDatanode()); @@ -641,13 +610,15 @@ public class DatanodeDescriptor extends DatanodeInfo { } DatanodeStorageInfo updateStorage(DatanodeStorage s) { - DatanodeStorageInfo storage = getStorageInfo(s.getStorageID()); - if (storage == null) { - storage = new DatanodeStorageInfo(this, s); - storageMap.put(s.getStorageID(), storage); - } else { - storage.setState(s.getState()); + synchronized (storageMap) { + DatanodeStorageInfo storage = storageMap.get(s.getStorageID()); + if (storage == null) { + storage = new DatanodeStorageInfo(this, s); + storageMap.put(s.getStorageID(), storage); + } else { + storage.setState(s.getState()); + } + return storage; } - return storage; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 9c8a1f07a30..e89dd585553 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -713,8 +713,10 @@ public class DatanodeManager { /** Start decommissioning the specified datanode. */ private void startDecommission(DatanodeDescriptor node) { if (!node.isDecommissionInProgress() && !node.isDecommissioned()) { - LOG.info("Start Decommissioning " + node + " with " + - node.numBlocks() + " blocks"); + for (DatanodeStorageInfo storage : node.getStorageInfos()) { + LOG.info("Start Decommissioning " + node + " " + storage + + " with " + storage.numBlocks() + " blocks"); + } heartbeatManager.startDecommission(node); node.decommissioningStatus.setStartTime(now()); @@ -1345,7 +1347,9 @@ public class DatanodeManager { LOG.info("Marking all datandoes as stale"); synchronized (datanodeMap) { for (DatanodeDescriptor dn : datanodeMap.values()) { - dn.markStaleAfterFailover(); + for(DatanodeStorageInfo storage : dn.getStorageInfos()) { + storage.markStaleAfterFailover(); + } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index 53fcde1c691..ba0e142b36c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -78,14 +78,62 @@ public class DatanodeStorageInfo { private long dfsUsed; private long remaining; private volatile BlockInfo blockList = null; - + private int numBlocks = 0; + + /** The number of block reports received */ + private int blockReportCount = 0; + + /** + * Set to false on any NN failover, and reset to true + * whenever a block report is received. + */ + private boolean heartbeatedSinceFailover = false; + + /** + * At startup or at failover, the storages in the cluster may have pending + * block deletions from a previous incarnation of the NameNode. The block + * contents are considered as stale until a block report is received. When a + * storage is considered as stale, the replicas on it are also considered as + * stale. If any block has at least one stale replica, then no invalidations + * will be processed for this block. See HDFS-1972. + */ + private boolean blockContentsStale = true; + public DatanodeStorageInfo(DatanodeDescriptor dn, DatanodeStorage s) { this.dn = dn; this.storageID = s.getStorageID(); this.storageType = s.getStorageType(); this.state = s.getState(); } - + + int getBlockReportCount() { + return blockReportCount; + } + + void setBlockReportCount(int blockReportCount) { + this.blockReportCount = blockReportCount; + } + + public boolean areBlockContentsStale() { + return blockContentsStale; + } + + public void markStaleAfterFailover() { + heartbeatedSinceFailover = false; + blockContentsStale = true; + } + + public void receivedHeartbeat() { + heartbeatedSinceFailover = true; + } + + public void receivedBlockReport() { + if (heartbeatedSinceFailover) { + blockContentsStale = false; + } + blockReportCount++; + } + public void setUtilization(long capacity, long dfsUsed, long remaining) { this.capacity = capacity; this.dfsUsed = dfsUsed; @@ -127,16 +175,22 @@ public class DatanodeStorageInfo { return false; // add to the head of the data-node list blockList = b.listInsert(blockList, this); + numBlocks++; return true; } public boolean removeBlock(BlockInfo b) { blockList = b.listRemove(blockList, this); - return b.removeStorage(this); + if (b.removeStorage(this)) { + numBlocks--; + return true; + } else { + return false; + } } public int numBlocks() { - return blockList == null ? 0 : blockList.listCount(this); + return numBlocks; } Iterator getBlockIterator() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index c63f4b56e66..6745cd7685a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -90,7 +90,6 @@ import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.IncorrectVersionException; import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; @@ -979,8 +978,6 @@ class NameNodeRpcServer implements NamenodeProtocols { bm.processReport(nodeReg, r.getStorage(), poolId, blocks); } - DatanodeDescriptor datanode = bm.getDatanodeManager().getDatanode(nodeReg); - datanode.receivedBlockReport(); if (nn.getFSImage().isUpgradeFinalized() && !nn.isStandbyState()) return new FinalizeCommand(poolId); return null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java index 2f659417479..243a2917f8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import java.io.IOException; import java.util.Collection; import java.util.HashSet; -import java.util.Iterator; import java.util.Set; import org.apache.hadoop.hdfs.DFSConfigKeys; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 1ddb91cfe1d..2eaf9a0b128 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -523,33 +523,30 @@ public class TestBlockManager { bm.getDatanodeManager().registerDatanode(nodeReg); bm.getDatanodeManager().addDatanode(node); // swap in spy assertEquals(node, bm.getDatanodeManager().getDatanode(node)); - assertTrue(node.isFirstBlockReport()); + assertEquals(0, ds.getBlockReportCount()); // send block report, should be processed reset(node); bm.processReport(node, new DatanodeStorage(ds.getStorageID()), "pool", new BlockListAsLongs(null, null)); - verify(node).receivedBlockReport(); - assertFalse(node.isFirstBlockReport()); + assertEquals(1, ds.getBlockReportCount()); // send block report again, should NOT be processed reset(node); bm.processReport(node, new DatanodeStorage(ds.getStorageID()), "pool", new BlockListAsLongs(null, null)); - verify(node, never()).receivedBlockReport(); - assertFalse(node.isFirstBlockReport()); + assertEquals(1, ds.getBlockReportCount()); // re-register as if node restarted, should update existing node bm.getDatanodeManager().removeDatanode(node); reset(node); bm.getDatanodeManager().registerDatanode(nodeReg); verify(node).updateRegInfo(nodeReg); - assertTrue(node.isFirstBlockReport()); // ready for report again + assertEquals(0, ds.getBlockReportCount()); // ready for report again // send block report, should be processed after restart reset(node); bm.processReport(node, new DatanodeStorage(ds.getStorageID()), "pool", new BlockListAsLongs(null, null)); - verify(node).receivedBlockReport(); - assertFalse(node.isFirstBlockReport()); + assertEquals(1, ds.getBlockReportCount()); } @Test @@ -570,13 +567,12 @@ public class TestBlockManager { bm.getDatanodeManager().registerDatanode(nodeReg); bm.getDatanodeManager().addDatanode(node); // swap in spy assertEquals(node, bm.getDatanodeManager().getDatanode(node)); - assertTrue(node.isFirstBlockReport()); + assertEquals(0, ds.getBlockReportCount()); // send block report while pretending to already have blocks reset(node); doReturn(1).when(node).numBlocks(); bm.processReport(node, new DatanodeStorage(ds.getStorageID()), "pool", new BlockListAsLongs(null, null)); - verify(node).receivedBlockReport(); - assertFalse(node.isFirstBlockReport()); + assertEquals(1, ds.getBlockReportCount()); } } From b2976af14034c6e2a7e9964535b9f363bfc31150 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 11 Sep 2013 06:30:01 +0000 Subject: [PATCH 11/76] HDFS-5157. Add StorageType to FsVolume. Contributed by Junping Du git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1521743 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 ++ .../hadoop/hdfs/server/datanode/DataNode.java | 7 ++---- .../datanode/fsdataset/FsVolumeSpi.java | 4 ++++ .../fsdataset/impl/FsDatasetImpl.java | 22 +++++++++++++++++-- .../datanode/fsdataset/impl/FsVolumeImpl.java | 10 ++++++++- .../datanode/fsdataset/impl/FsVolumeList.java | 1 + .../server/datanode/TestDirectoryScanner.java | 6 +++++ 7 files changed, 44 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 49379bc1f9f..7980b95c568 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -21,3 +21,5 @@ IMPROVEMENTS: HDFS-5134. Move blockContentsStale, heartbeatedSinceFailover and firstBlockReport from DatanodeDescriptor to DatanodeStorageInfo; and fix a synchronization problem in DatanodeStorageInfo. (szetszwo) + + HDFS-5157. Add StorageType to FsVolume. (Junping Du via szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index cfb323c527e..0769d9c657e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -52,7 +52,6 @@ import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.DataOutputStream; -import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -62,7 +61,6 @@ import java.net.InetSocketAddress; import java.net.Socket; import java.net.SocketException; import java.net.SocketTimeoutException; -import java.net.URI; import java.net.UnknownHostException; import java.nio.channels.ClosedByInterruptException; import java.nio.channels.SocketChannel; @@ -122,7 +120,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.common.StorageInfo; -import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; @@ -160,7 +157,6 @@ import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.JvmPauseMonitor; import org.apache.hadoop.util.ServicePlugin; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Time; import org.apache.hadoop.util.VersionInfo; import org.mortbay.util.ajax.JSON; @@ -1709,7 +1705,8 @@ public class DataNode extends Configured return locations; } - static Collection getStorageLocations(Configuration conf) { + public static Collection getStorageLocations( + Configuration conf) { return parseStorageLocations( conf.getTrimmedStringCollection(DFS_DATANODE_DATA_DIR_KEY)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java index 58cfe2e8228..c0d71303876 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java @@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset; import java.io.File; import java.io.IOException; +import org.apache.hadoop.hdfs.StorageType; + /** * This is an interface for the underlying volume. */ @@ -38,4 +40,6 @@ public interface FsVolumeSpi { /** @return the directory for the finalized blocks in the block pool. */ public File getFinalizedDir(String bpid) throws IOException; + + public StorageType getStorageType(); } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 2a210b2bc7c..5218dd75db1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -44,6 +44,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; @@ -65,6 +66,7 @@ import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo; import org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException; import org.apache.hadoop.hdfs.server.datanode.ReplicaUnderRecovery; import org.apache.hadoop.hdfs.server.datanode.ReplicaWaitingToBeRecovered; +import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream; @@ -188,6 +190,7 @@ class FsDatasetImpl implements FsDatasetSpi { DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_DEFAULT); String[] dataDirs = conf.getTrimmedStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY); + Collection dataLocations = DataNode.getStorageLocations(conf); int volsConfigured = (dataDirs == null) ? 0 : dataDirs.length; int volsFailed = volsConfigured - storage.getNumStorageDirs(); @@ -209,8 +212,13 @@ class FsDatasetImpl implements FsDatasetSpi { storage.getNumStorageDirs()); for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) { final File dir = storage.getStorageDir(idx).getCurrentDir(); - volArray.add(new FsVolumeImpl(this, storage.getStorageID(), dir, conf)); - LOG.info("Added volume - " + dir); + // TODO: getStorageTypeFromLocations() is only a temporary workaround and + // should be replaced with getting storage type from DataStorage (missing + // storage type now) directly. + final StorageType storageType = getStorageTypeFromLocations(dataLocations, dir); + volArray.add(new FsVolumeImpl(this, storage.getStorageID(), dir, conf, + storageType)); + LOG.info("Added volume - " + dir + ", StorageType: " + storageType); } volumeMap = new ReplicaMap(this); @@ -231,6 +239,16 @@ class FsDatasetImpl implements FsDatasetSpi { registerMBean(storage.getStorageID()); } + private StorageType getStorageTypeFromLocations( + Collection dataLocations, File dir) { + for (StorageLocation dataLocation : dataLocations) { + if (dataLocation.getFile().equals(dir)) { + return dataLocation.getStorageType(); + } + } + return StorageType.DEFAULT; + } + /** * Return the total space used by dfs datanode */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index 95cc74e5a5b..05dc7bdfef8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.DF; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.datanode.DataStorage; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; @@ -43,6 +44,7 @@ import org.apache.hadoop.util.DiskChecker.DiskErrorException; class FsVolumeImpl implements FsVolumeSpi { private final FsDatasetImpl dataset; private final String storageID; + private final StorageType storageType; private final Map bpSlices = new HashMap(); private final File currentDir; // /current @@ -50,7 +52,7 @@ class FsVolumeImpl implements FsVolumeSpi { private final long reserved; FsVolumeImpl(FsDatasetImpl dataset, String storageID, File currentDir, - Configuration conf) throws IOException { + Configuration conf, StorageType storageType) throws IOException { this.dataset = dataset; this.storageID = storageID; this.reserved = conf.getLong( @@ -59,6 +61,7 @@ class FsVolumeImpl implements FsVolumeSpi { this.currentDir = currentDir; File parent = currentDir.getParentFile(); this.usage = new DF(parent, conf); + this.storageType = storageType; } File getCurrentDir() { @@ -290,4 +293,9 @@ class FsVolumeImpl implements FsVolumeSpi { String getStorageID() { return storageID; } + + @Override + public StorageType getStorageType() { + return storageType; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index e9d11da5cb2..90d2aee8e48 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -56,6 +56,7 @@ class FsVolumeList { * @param blockSize free space needed on the volume * @return next volume to store the block in. */ + // TODO this will be replaced by getting volume from StorageID directly later. synchronized FsVolumeImpl getNextVolume(long blockSize) throws IOException { return blockChooser.chooseVolume(volumes, blockSize); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java index c5decf2eaa6..cec8a15daa4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; @@ -406,6 +407,11 @@ public class TestDirectoryScanner { public File getFinalizedDir(String bpid) throws IOException { return new File("/base/current/" + bpid + "/finalized"); } + + @Override + public StorageType getStorageType() { + return StorageType.DEFAULT; + } } private final static TestFsVolumeSpi TEST_VOLUME = new TestFsVolumeSpi(); From abf09f090f77a7e54e331b7a07354e7926b60dc9 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 18 Sep 2013 15:12:52 +0000 Subject: [PATCH 12/76] HDFS-4990. Change BlockPlacementPolicy to choose storages instead of datanodes. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1524444 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 + .../apache/hadoop/hdfs/DFSOutputStream.java | 3 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 17 + .../hadoop/hdfs/protocol/ClientProtocol.java | 3 +- .../hadoop/hdfs/protocol/LocatedBlock.java | 22 +- ...amenodeProtocolServerSideTranslatorPB.java | 9 +- .../ClientNamenodeProtocolTranslatorPB.java | 4 +- .../hadoop/hdfs/protocolPB/PBHelper.java | 24 +- .../hadoop/hdfs/server/balancer/Balancer.java | 13 +- .../server/blockmanagement/BlockManager.java | 63 ++-- .../blockmanagement/BlockPlacementPolicy.java | 19 +- .../BlockPlacementPolicyDefault.java | 324 ++++++++++-------- .../BlockPlacementPolicyWithNodeGroup.java | 115 ++++--- .../blockmanagement/DatanodeDescriptor.java | 37 +- .../blockmanagement/DatanodeStorageInfo.java | 36 +- .../PendingReplicationBlocks.java | 22 +- .../hdfs/server/namenode/FSNamesystem.java | 37 +- .../server/namenode/NameNodeRpcServer.java | 7 +- .../web/resources/NamenodeWebHdfsMethods.java | 14 +- .../hdfs/server/protocol/BlockCommand.java | 26 +- .../main/proto/ClientNamenodeProtocol.proto | 1 + .../src/main/proto/DatanodeProtocol.proto | 2 + .../hadoop-hdfs/src/main/proto/hdfs.proto | 7 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 30 +- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 3 +- .../blockmanagement/BlockManagerTestUtil.java | 10 +- .../blockmanagement/TestBlockManager.java | 131 ++++--- .../TestDatanodeDescriptor.java | 7 +- .../TestHeartbeatHandling.java | 14 +- .../TestPendingReplication.java | 34 +- .../TestReplicationPolicy.java | 258 ++++++++------ .../TestReplicationPolicyWithNodeGroup.java | 307 +++++++++-------- 32 files changed, 926 insertions(+), 676 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 7980b95c568..b9d4981a816 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -23,3 +23,6 @@ IMPROVEMENTS: fix a synchronization problem in DatanodeStorageInfo. (szetszwo) HDFS-5157. Add StorageType to FsVolume. (Junping Du via szetszwo) + + HDFS-4990. Change BlockPlacementPolicy to choose storages instead of + datanodes. (szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index a9bcc16eb44..e8678f17a2e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -913,7 +913,8 @@ public class DFSOutputStream extends FSOutputSummer //get a new datanode final DatanodeInfo[] original = nodes; final LocatedBlock lb = dfsClient.namenode.getAdditionalDatanode( - src, block, nodes, failed.toArray(new DatanodeInfo[failed.size()]), + src, block, nodes, storageIDs, + failed.toArray(new DatanodeInfo[failed.size()]), 1, dfsClient.clientName); nodes = lb.getLocations(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index e3b61abc0bd..6b7013e158d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -131,6 +131,23 @@ public class DFSUtil { return SECURE_RANDOM.get(); } + /** Shuffle the elements in the given array. */ + public static T[] shuffle(final T[] array) { + if (array != null && array.length > 0) { + final Random random = getRandom(); + for (int n = array.length; n > 1; ) { + final int randomIndex = random.nextInt(n); + n--; + if (n != randomIndex) { + final T tmp = array[randomIndex]; + array[randomIndex] = array[n]; + array[n] = tmp; + } + } + } + return array; + } + /** * Compartor for sorting DataNodeInfo[] based on decommissioned states. * Decommissioned nodes are moved to the end of the array on sorting with diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 66b6fbb0d90..b2b45f3dbf2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -353,7 +353,8 @@ public interface ClientProtocol { */ @Idempotent public LocatedBlock getAdditionalDatanode(final String src, final ExtendedBlock blk, - final DatanodeInfo[] existings, final DatanodeInfo[] excludes, + final DatanodeInfo[] existings, final String[] existingStorageIDs, + final DatanodeInfo[] excludes, final int numAdditionalNodes, final String clientName ) throws AccessControlException, FileNotFoundException, SafeModeException, UnresolvedLinkException, IOException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java index e26ca81d073..78ce1365a91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java @@ -59,18 +59,18 @@ public class LocatedBlock { this(b, locs, null, null, startOffset, corrupt); } - public static LocatedBlock createLocatedBlock(ExtendedBlock b, - DatanodeStorageInfo[] storages, long startOffset, boolean corrupt) { - final DatanodeInfo[] locs = new DatanodeInfo[storages.length]; - final String[] storageIDs = new String[storages.length]; - final StorageType[] storageType = new StorageType[storages.length]; - for(int i = 0; i < storages.length; i++) { - locs[i] = storages[i].getDatanodeDescriptor(); - storageIDs[i] = storages[i].getStorageID(); - storageType[i] = storages[i].getStorageType(); - } - return new LocatedBlock(b, locs, storageIDs, storageType, startOffset, corrupt); + public LocatedBlock(ExtendedBlock b, DatanodeStorageInfo[] storages) { + this(b, storages, -1, false); // startOffset is unknown } + + public LocatedBlock(ExtendedBlock b, DatanodeStorageInfo[] storages, + long startOffset, boolean corrupt) { + this(b, DatanodeStorageInfo.toDatanodeInfos(storages), + DatanodeStorageInfo.toStorageIDs(storages), + DatanodeStorageInfo.toStorageTypes(storages), + startOffset, corrupt); // startOffset is unknown + } + public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, String[] storageIDs, StorageType[] storageTypes, long startOffset, boolean corrupt) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 76ba6cab59a..6e8e7bc34eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -405,14 +405,17 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements throws ServiceException { try { List existingList = req.getExistingsList(); + List existingStorageIDsList = req.getExistingStorageIDsList(); List excludesList = req.getExcludesList(); - LocatedBlock result = server.getAdditionalDatanode( - req.getSrc(), PBHelper.convert(req.getBlk()), + LocatedBlock result = server.getAdditionalDatanode(req.getSrc(), + PBHelper.convert(req.getBlk()), PBHelper.convert(existingList.toArray( new DatanodeInfoProto[existingList.size()])), + existingStorageIDsList.toArray( + new String[existingStorageIDsList.size()]), PBHelper.convert(excludesList.toArray( new DatanodeInfoProto[excludesList.size()])), - req.getNumAdditionalNodes(), req.getClientName()); + req.getNumAdditionalNodes(), req.getClientName()); return GetAdditionalDatanodeResponseProto.newBuilder().setBlock( PBHelper.convert(result)) .build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 7bcd014372f..84d10e473a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -335,7 +335,8 @@ public class ClientNamenodeProtocolTranslatorPB implements @Override public LocatedBlock getAdditionalDatanode(String src, ExtendedBlock blk, - DatanodeInfo[] existings, DatanodeInfo[] excludes, + DatanodeInfo[] existings, String[] existingStorageIDs, + DatanodeInfo[] excludes, int numAdditionalNodes, String clientName) throws AccessControlException, FileNotFoundException, SafeModeException, UnresolvedLinkException, IOException { @@ -344,6 +345,7 @@ public class ClientNamenodeProtocolTranslatorPB implements .setSrc(src) .setBlk(PBHelper.convert(blk)) .addAllExistings(PBHelper.convert(existings)) + .addAllExistingStorageIDs(Arrays.asList(existingStorageIDs)) .addAllExcludes(PBHelper.convert(excludes)) .setNumAdditionalNodes(numAdditionalNodes) .setClientName(clientName) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index c82df6f8290..d118904d0ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -94,6 +94,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.File import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto.Builder; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlocksProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageIDsProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeCommandProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto; @@ -744,7 +745,8 @@ public class PBHelper { for (int i = 0; i < blocks.length; i++) { builder.addBlocks(PBHelper.convert(blocks[i])); } - builder.addAllTargets(PBHelper.convert(cmd.getTargets())); + builder.addAllTargets(convert(cmd.getTargets())) + .addAllTargetStorageIDs(convert(cmd.getTargetStorageIDs())); return builder.build(); } @@ -757,6 +759,15 @@ public class PBHelper { return Arrays.asList(ret); } + private static List convert(String[][] targetStorageIDs) { + StorageIDsProto[] ret = new StorageIDsProto[targetStorageIDs.length]; + for (int i = 0; i < targetStorageIDs.length; i++) { + ret[i] = StorageIDsProto.newBuilder() + .addAllStorageIDs(Arrays.asList(targetStorageIDs[i])).build(); + } + return Arrays.asList(ret); + } + public static DatanodeCommandProto convert(DatanodeCommand datanodeCommand) { DatanodeCommandProto.Builder builder = DatanodeCommandProto.newBuilder(); if (datanodeCommand == null) { @@ -831,6 +842,14 @@ public class PBHelper { for (int i = 0; i < targetList.size(); i++) { targets[i] = PBHelper.convert(targetList.get(i)); } + + List targetStorageIDsList = blkCmd.getTargetStorageIDsList(); + String[][] targetStorageIDs = new String[targetStorageIDsList.size()][]; + for(int i = 0; i < targetStorageIDs.length; i++) { + List storageIDs = targetStorageIDsList.get(i).getStorageIDsList(); + targetStorageIDs[i] = storageIDs.toArray(new String[storageIDs.size()]); + } + int action = DatanodeProtocol.DNA_UNKNOWN; switch (blkCmd.getAction()) { case TRANSFER: @@ -843,7 +862,8 @@ public class PBHelper { action = DatanodeProtocol.DNA_SHUTDOWN; break; } - return new BlockCommand(action, blkCmd.getBlockPoolId(), blocks, targets); + return new BlockCommand(action, blkCmd.getBlockPoolId(), blocks, targets, + targetStorageIDs); } public static DatanodeInfo[] convert(DatanodeInfosProto datanodeInfosProto) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 1ed0e1915f4..27cf20a0a07 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -835,16 +835,6 @@ public class Balancer { DFSConfigKeys.DFS_BALANCER_DISPATCHERTHREADS_DEFAULT)); } - /* Shuffle datanode array */ - static private void shuffleArray(DatanodeInfo[] datanodes) { - for (int i=datanodes.length; i>1; i--) { - int randomIndex = DFSUtil.getRandom().nextInt(i); - DatanodeInfo tmp = datanodes[randomIndex]; - datanodes[randomIndex] = datanodes[i-1]; - datanodes[i-1] = tmp; - } - } - /* Given a data node set, build a network topology and decide * over-utilized datanodes, above average utilized datanodes, * below average utilized datanodes, and underutilized datanodes. @@ -874,8 +864,7 @@ public class Balancer { * an increasing order or a decreasing order. */ long overLoadedBytes = 0L, underLoadedBytes = 0L; - shuffleArray(datanodes); - for (DatanodeInfo datanode : datanodes) { + for (DatanodeInfo datanode : DFSUtil.shuffle(datanodes)) { if (datanode.isDecommissioned() || datanode.isDecommissionInProgress()) { continue; // ignore decommissioning or decommissioned nodes } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 87a5fc976e2..094c791b1df 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -45,6 +45,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportIterator; @@ -473,8 +474,8 @@ public class BlockManager { private void dumpBlockMeta(Block block, PrintWriter out) { List containingNodes = new ArrayList(); - List containingLiveReplicasNodes = - new ArrayList(); + List containingLiveReplicasNodes = + new ArrayList(); NumberReplicas numReplicas = new NumberReplicas(); // source node returned is not used @@ -774,7 +775,7 @@ public class BlockManager { final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)blk; final DatanodeStorageInfo[] storages = uc.getExpectedStorageLocations(); final ExtendedBlock eb = new ExtendedBlock(namesystem.getBlockPoolId(), blk); - return LocatedBlock.createLocatedBlock(eb, storages, pos, false); + return new LocatedBlock(eb, storages, pos, false); } // get block locations @@ -789,14 +790,14 @@ public class BlockManager { final int numNodes = blocksMap.numNodes(blk); final boolean isCorrupt = numCorruptNodes == numNodes; final int numMachines = isCorrupt ? numNodes: numNodes - numCorruptNodes; - final DatanodeDescriptor[] machines = new DatanodeDescriptor[numMachines]; + final DatanodeStorageInfo[] machines = new DatanodeStorageInfo[numMachines]; int j = 0; if (numMachines > 0) { for(DatanodeStorageInfo storage : blocksMap.getStorages(blk)) { final DatanodeDescriptor d = storage.getDatanodeDescriptor(); final boolean replicaCorrupt = corruptReplicas.isReplicaCorrupt(blk, d); if (isCorrupt || (!isCorrupt && !replicaCorrupt)) - machines[j++] = d; + machines[j++] = storage; } } assert j == machines.length : @@ -1195,7 +1196,7 @@ public class BlockManager { @VisibleForTesting int computeReplicationWorkForBlocks(List> blocksToReplicate) { int requiredReplication, numEffectiveReplicas; - List containingNodes, liveReplicaNodes; + List containingNodes; DatanodeDescriptor srcNode; BlockCollection bc = null; int additionalReplRequired; @@ -1220,7 +1221,7 @@ public class BlockManager { // get a source data-node containingNodes = new ArrayList(); - liveReplicaNodes = new ArrayList(); + List liveReplicaNodes = new ArrayList(); NumberReplicas numReplicas = new NumberReplicas(); srcNode = chooseSourceDatanode( block, containingNodes, liveReplicaNodes, numReplicas, @@ -1279,7 +1280,7 @@ public class BlockManager { namesystem.writeLock(); try { for(ReplicationWork rw : work){ - DatanodeDescriptor[] targets = rw.targets; + final DatanodeStorageInfo[] targets = rw.targets; if(targets == null || targets.length == 0){ rw.targets = null; continue; @@ -1317,7 +1318,8 @@ public class BlockManager { if ( (numReplicas.liveReplicas() >= requiredReplication) && (!blockHasEnoughRacks(block)) ) { - if (rw.srcNode.getNetworkLocation().equals(targets[0].getNetworkLocation())) { + if (rw.srcNode.getNetworkLocation().equals( + targets[0].getDatanodeDescriptor().getNetworkLocation())) { //No use continuing, unless a new rack in this case continue; } @@ -1327,8 +1329,8 @@ public class BlockManager { rw.srcNode.addBlockToBeReplicated(block, targets); scheduledWork++; - for (DatanodeDescriptor dn : targets) { - dn.incBlocksScheduled(); + for (DatanodeStorageInfo storage : targets) { + storage.getDatanodeDescriptor().incBlocksScheduled(); } // Move the block-replication into a "pending" state. @@ -1354,7 +1356,7 @@ public class BlockManager { if (blockLog.isInfoEnabled()) { // log which blocks have been scheduled for replication for(ReplicationWork rw : work){ - DatanodeDescriptor[] targets = rw.targets; + DatanodeStorageInfo[] targets = rw.targets; if (targets != null && targets.length != 0) { StringBuilder targetList = new StringBuilder("datanode(s)"); for (int k = 0; k < targets.length; k++) { @@ -1383,15 +1385,16 @@ public class BlockManager { * @see BlockPlacementPolicy#chooseTarget(String, int, Node, * List, boolean, Set, long) */ - public DatanodeDescriptor[] chooseTarget(final String src, + public DatanodeStorageInfo[] chooseTarget(final String src, final int numOfReplicas, final DatanodeDescriptor client, final Set excludedNodes, final long blocksize, List favoredNodes) throws IOException { List favoredDatanodeDescriptors = getDatanodeDescriptors(favoredNodes); - final DatanodeDescriptor targets[] = blockplacement.chooseTarget(src, + final DatanodeStorageInfo[] targets = blockplacement.chooseTarget(src, numOfReplicas, client, excludedNodes, blocksize, - favoredDatanodeDescriptors); + // TODO: get storage type from file + favoredDatanodeDescriptors, StorageType.DEFAULT); if (targets.length < minReplication) { throw new IOException("File " + src + " could only be replicated to " + targets.length + " nodes instead of minReplication (=" @@ -1452,12 +1455,11 @@ public class BlockManager { * the given block */ @VisibleForTesting - DatanodeDescriptor chooseSourceDatanode( - Block block, - List containingNodes, - List nodesContainingLiveReplicas, - NumberReplicas numReplicas, - int priority) { + DatanodeDescriptor chooseSourceDatanode(Block block, + List containingNodes, + List nodesContainingLiveReplicas, + NumberReplicas numReplicas, + int priority) { containingNodes.clear(); nodesContainingLiveReplicas.clear(); DatanodeDescriptor srcNode = null; @@ -1478,7 +1480,7 @@ public class BlockManager { else if (excessBlocks != null && excessBlocks.contains(block)) { excess++; } else { - nodesContainingLiveReplicas.add(node); + nodesContainingLiveReplicas.add(storage); live++; } containingNodes.add(node); @@ -1621,7 +1623,8 @@ public class BlockManager { // To minimize startup time, we discard any second (or later) block reports // that we receive while still in startup phase. - final DatanodeStorageInfo storageInfo = node.getStorageInfo(storage.getStorageID()); + final DatanodeStorageInfo storageInfo = node.updateStorage(storage); + LOG.info("XXX storageInfo=" + storageInfo + ", storage=" + storage); if (namesystem.isInStartupSafeMode() && storageInfo.getBlockReportCount() > 0) { blockLog.info("BLOCK* processReport: " @@ -2636,7 +2639,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block // // Modify the blocks->datanode map and node's map. // - pendingReplications.decrement(block, node); + pendingReplications.decrement(block, node, storageID); processAndHandleReportedBlock(node, storageID, block, ReplicaState.FINALIZED, delHintNode); } @@ -3225,24 +3228,24 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block private DatanodeDescriptor srcNode; private List containingNodes; - private List liveReplicaNodes; + private List liveReplicaStorages; private int additionalReplRequired; - private DatanodeDescriptor targets[]; + private DatanodeStorageInfo targets[]; private int priority; public ReplicationWork(Block block, BlockCollection bc, DatanodeDescriptor srcNode, List containingNodes, - List liveReplicaNodes, + List liveReplicaStorages, int additionalReplRequired, int priority) { this.block = block; this.bc = bc; this.srcNode = srcNode; this.containingNodes = containingNodes; - this.liveReplicaNodes = liveReplicaNodes; + this.liveReplicaStorages = liveReplicaStorages; this.additionalReplRequired = additionalReplRequired; this.priority = priority; this.targets = null; @@ -3251,8 +3254,8 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block private void chooseTargets(BlockPlacementPolicy blockplacement, Set excludedNodes) { targets = blockplacement.chooseTarget(bc.getName(), - additionalReplRequired, srcNode, liveReplicaNodes, false, - excludedNodes, block.getNumBytes()); + additionalReplRequired, srcNode, liveReplicaStorages, false, + excludedNodes, block.getNumBytes(), StorageType.DEFAULT); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java index 48f77298dbb..5d9262d6022 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -67,13 +68,14 @@ public abstract class BlockPlacementPolicy { * @return array of DatanodeDescriptor instances chosen as target * and sorted as a pipeline. */ - public abstract DatanodeDescriptor[] chooseTarget(String srcPath, + public abstract DatanodeStorageInfo[] chooseTarget(String srcPath, int numOfReplicas, Node writer, - List chosenNodes, + List chosen, boolean returnChosenNodes, Set excludedNodes, - long blocksize); + long blocksize, + StorageType storageType); /** * Same as {@link #chooseTarget(String, int, Node, List, boolean, @@ -82,16 +84,19 @@ public abstract class BlockPlacementPolicy { * is only a hint and due to cluster state, namenode may not be * able to place the blocks on these datanodes. */ - DatanodeDescriptor[] chooseTarget(String src, + DatanodeStorageInfo[] chooseTarget(String src, int numOfReplicas, Node writer, Set excludedNodes, - long blocksize, List favoredNodes) { + long blocksize, + List favoredNodes, + StorageType storageType) { // This class does not provide the functionality of placing // a block in favored datanodes. The implementations of this class // are expected to provide this functionality + return chooseTarget(src, numOfReplicas, writer, - new ArrayList(numOfReplicas), false, excludedNodes, - blocksize); + new ArrayList(numOfReplicas), false, + excludedNodes, blocksize, storageType); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 7fb4cf01f7d..af786e716dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -29,11 +29,14 @@ import java.util.TreeSet; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.namenode.FSClusterStats; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; @@ -103,99 +106,101 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { } @Override - public DatanodeDescriptor[] chooseTarget(String srcPath, + public DatanodeStorageInfo[] chooseTarget(String srcPath, int numOfReplicas, Node writer, - List chosenNodes, + List chosenNodes, boolean returnChosenNodes, Set excludedNodes, - long blocksize) { + long blocksize, + StorageType storageType) { return chooseTarget(numOfReplicas, writer, chosenNodes, returnChosenNodes, - excludedNodes, blocksize); + excludedNodes, blocksize, storageType); } @Override - DatanodeDescriptor[] chooseTarget(String src, + DatanodeStorageInfo[] chooseTarget(String src, int numOfReplicas, Node writer, Set excludedNodes, long blocksize, - List favoredNodes) { + List favoredNodes, + StorageType storageType) { try { if (favoredNodes == null || favoredNodes.size() == 0) { // Favored nodes not specified, fall back to regular block placement. return chooseTarget(src, numOfReplicas, writer, - new ArrayList(numOfReplicas), false, - excludedNodes, blocksize); + new ArrayList(numOfReplicas), false, + excludedNodes, blocksize, storageType); } Set favoriteAndExcludedNodes = excludedNodes == null ? new HashSet() : new HashSet(excludedNodes); // Choose favored nodes - List results = new ArrayList(); + List results = new ArrayList(); boolean avoidStaleNodes = stats != null && stats.isAvoidingStaleDataNodesForWrite(); for (int i = 0; i < Math.min(favoredNodes.size(), numOfReplicas); i++) { DatanodeDescriptor favoredNode = favoredNodes.get(i); // Choose a single node which is local to favoredNode. // 'results' is updated within chooseLocalNode - DatanodeDescriptor target = chooseLocalNode(favoredNode, + final DatanodeStorageInfo target = chooseLocalStorage(favoredNode, favoriteAndExcludedNodes, blocksize, - getMaxNodesPerRack(results, - numOfReplicas)[1], results, avoidStaleNodes); + getMaxNodesPerRack(results.size(), numOfReplicas)[1], + results, avoidStaleNodes, storageType); if (target == null) { LOG.warn("Could not find a target for file " + src + " with favored node " + favoredNode); continue; } - favoriteAndExcludedNodes.add(target); + favoriteAndExcludedNodes.add(target.getDatanodeDescriptor()); } if (results.size() < numOfReplicas) { // Not enough favored nodes, choose other nodes. numOfReplicas -= results.size(); - DatanodeDescriptor[] remainingTargets = + DatanodeStorageInfo[] remainingTargets = chooseTarget(src, numOfReplicas, writer, results, - false, favoriteAndExcludedNodes, blocksize); + false, favoriteAndExcludedNodes, blocksize, storageType); for (int i = 0; i < remainingTargets.length; i++) { results.add(remainingTargets[i]); } } return getPipeline(writer, - results.toArray(new DatanodeDescriptor[results.size()])); + results.toArray(new DatanodeStorageInfo[results.size()])); } catch (NotEnoughReplicasException nr) { // Fall back to regular block placement disregarding favored nodes hint return chooseTarget(src, numOfReplicas, writer, - new ArrayList(numOfReplicas), false, - excludedNodes, blocksize); + new ArrayList(numOfReplicas), false, + excludedNodes, blocksize, storageType); } } /** This is the implementation. */ - private DatanodeDescriptor[] chooseTarget(int numOfReplicas, + private DatanodeStorageInfo[] chooseTarget(int numOfReplicas, Node writer, - List chosenNodes, + List chosenStorage, boolean returnChosenNodes, Set excludedNodes, - long blocksize) { + long blocksize, + StorageType storageType) { if (numOfReplicas == 0 || clusterMap.getNumOfLeaves()==0) { - return DatanodeDescriptor.EMPTY_ARRAY; + return DatanodeStorageInfo.EMPTY_ARRAY; } if (excludedNodes == null) { excludedNodes = new HashSet(); } - int[] result = getMaxNodesPerRack(chosenNodes, numOfReplicas); + int[] result = getMaxNodesPerRack(chosenStorage.size(), numOfReplicas); numOfReplicas = result[0]; int maxNodesPerRack = result[1]; - List results = - new ArrayList(chosenNodes); - for (DatanodeDescriptor node:chosenNodes) { + final List results = new ArrayList(chosenStorage); + for (DatanodeStorageInfo storage : chosenStorage) { // add localMachine and related nodes to excludedNodes - addToExcludedNodes(node, excludedNodes); + addToExcludedNodes(storage.getDatanodeDescriptor(), excludedNodes); } if (!clusterMap.contains(writer)) { @@ -205,20 +210,19 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { boolean avoidStaleNodes = (stats != null && stats.isAvoidingStaleDataNodesForWrite()); Node localNode = chooseTarget(numOfReplicas, writer, - excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes); + excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageType); if (!returnChosenNodes) { - results.removeAll(chosenNodes); + results.removeAll(chosenStorage); } // sorting nodes to form a pipeline return getPipeline((writer==null)?localNode:writer, - results.toArray(new DatanodeDescriptor[results.size()])); + results.toArray(new DatanodeStorageInfo[results.size()])); } - private int[] getMaxNodesPerRack(List chosenNodes, - int numOfReplicas) { + private int[] getMaxNodesPerRack(int numOfChosen, int numOfReplicas) { int clusterSize = clusterMap.getNumOfLeaves(); - int totalNumOfReplicas = chosenNodes.size()+numOfReplicas; + int totalNumOfReplicas = numOfChosen + numOfReplicas; if (totalNumOfReplicas > clusterSize) { numOfReplicas -= (totalNumOfReplicas-clusterSize); totalNumOfReplicas = clusterSize; @@ -243,8 +247,9 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { Set excludedNodes, long blocksize, int maxNodesPerRack, - List results, - final boolean avoidStaleNodes) { + List results, + final boolean avoidStaleNodes, + StorageType storageType) { if (numOfReplicas == 0 || clusterMap.getNumOfLeaves()==0) { return writer; } @@ -253,7 +258,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { int numOfResults = results.size(); boolean newBlock = (numOfResults==0); if ((writer == null || !(writer instanceof DatanodeDescriptor)) && !newBlock) { - writer = results.get(0); + writer = results.get(0).getDatanodeDescriptor(); } // Keep a copy of original excludedNodes @@ -261,42 +266,49 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { new HashSet(excludedNodes) : null; try { if (numOfResults == 0) { - writer = chooseLocalNode(writer, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + writer = chooseLocalStorage(writer, excludedNodes, blocksize, + maxNodesPerRack, results, avoidStaleNodes, storageType) + .getDatanodeDescriptor(); if (--numOfReplicas == 0) { return writer; } } + final DatanodeDescriptor dn0 = results.get(0).getDatanodeDescriptor(); if (numOfResults <= 1) { - chooseRemoteRack(1, results.get(0), excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + chooseRemoteRack(1, dn0, excludedNodes, blocksize, maxNodesPerRack, + results, avoidStaleNodes, storageType); if (--numOfReplicas == 0) { return writer; } } if (numOfResults <= 2) { - if (clusterMap.isOnSameRack(results.get(0), results.get(1))) { - chooseRemoteRack(1, results.get(0), excludedNodes, - blocksize, maxNodesPerRack, - results, avoidStaleNodes); + final DatanodeDescriptor dn1 = results.get(1).getDatanodeDescriptor(); + if (clusterMap.isOnSameRack(dn0, dn1)) { + chooseRemoteRack(1, dn0, excludedNodes, blocksize, maxNodesPerRack, + results, avoidStaleNodes, storageType); } else if (newBlock){ - chooseLocalRack(results.get(1), excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + chooseLocalRack(dn1, excludedNodes, blocksize, maxNodesPerRack, + results, avoidStaleNodes, storageType); } else { chooseLocalRack(writer, excludedNodes, blocksize, maxNodesPerRack, - results, avoidStaleNodes); + results, avoidStaleNodes, storageType); } if (--numOfReplicas == 0) { return writer; } } chooseRandom(numOfReplicas, NodeBase.ROOT, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + maxNodesPerRack, results, avoidStaleNodes, storageType); } catch (NotEnoughReplicasException e) { - LOG.warn("Not able to place enough replicas, still in need of " - + (totalReplicasExpected - results.size()) + " to reach " - + totalReplicasExpected + "\n" - + e.getMessage()); + final String message = "Failed to place enough replicas, still in need of " + + (totalReplicasExpected - results.size()) + " to reach " + + totalReplicasExpected + "."; + if (LOG.isTraceEnabled()) { + LOG.trace(message, e); + } else { + LOG.warn(message + " " + e.getMessage()); + } + if (avoidStaleNodes) { // Retry chooseTarget again, this time not avoiding stale nodes. @@ -304,14 +316,14 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { // not chosen because they were stale, decommissioned, etc. // We need to additionally exclude the nodes that were added to the // result list in the successful calls to choose*() above. - for (Node node : results) { - oldExcludedNodes.add(node); + for (DatanodeStorageInfo resultStorage : results) { + oldExcludedNodes.add(resultStorage.getDatanodeDescriptor()); } // Set numOfReplicas, since it can get out of sync with the result list // if the NotEnoughReplicasException was thrown in chooseRandom(). numOfReplicas = totalReplicasExpected - results.size(); return chooseTarget(numOfReplicas, writer, oldExcludedNodes, blocksize, - maxNodesPerRack, results, false); + maxNodesPerRack, results, false, storageType); } } return writer; @@ -321,32 +333,36 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { * Choose localMachine as the target. * if localMachine is not available, * choose a node on the same rack - * @return the chosen node + * @return the chosen storage */ - protected DatanodeDescriptor chooseLocalNode(Node localMachine, + protected DatanodeStorageInfo chooseLocalStorage(Node localMachine, Set excludedNodes, long blocksize, int maxNodesPerRack, - List results, - boolean avoidStaleNodes) + List results, + boolean avoidStaleNodes, + StorageType storageType) throws NotEnoughReplicasException { // if no local machine, randomly choose one node if (localMachine == null) return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + maxNodesPerRack, results, avoidStaleNodes, storageType); if (preferLocalNode && localMachine instanceof DatanodeDescriptor) { DatanodeDescriptor localDatanode = (DatanodeDescriptor) localMachine; // otherwise try local machine first if (excludedNodes.add(localMachine)) { // was not in the excluded list - if (addIfIsGoodTarget(localDatanode, excludedNodes, blocksize, - maxNodesPerRack, false, results, avoidStaleNodes) >= 0) { - return localDatanode; + for(DatanodeStorageInfo localStorage : DFSUtil.shuffle( + localDatanode.getStorageInfos())) { + if (addIfIsGoodTarget(localStorage, excludedNodes, blocksize, + maxNodesPerRack, false, results, avoidStaleNodes, storageType) >= 0) { + return localStorage; + } } } } // try a node on local rack return chooseLocalRack(localMachine, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + maxNodesPerRack, results, avoidStaleNodes, storageType); } /** @@ -368,27 +384,29 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { * in the cluster. * @return the chosen node */ - protected DatanodeDescriptor chooseLocalRack(Node localMachine, + protected DatanodeStorageInfo chooseLocalRack(Node localMachine, Set excludedNodes, long blocksize, int maxNodesPerRack, - List results, - boolean avoidStaleNodes) + List results, + boolean avoidStaleNodes, + StorageType storageType) throws NotEnoughReplicasException { // no local machine, so choose a random machine if (localMachine == null) { return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + maxNodesPerRack, results, avoidStaleNodes, storageType); } // choose one from the local rack try { return chooseRandom(localMachine.getNetworkLocation(), excludedNodes, - blocksize, maxNodesPerRack, results, avoidStaleNodes); + blocksize, maxNodesPerRack, results, avoidStaleNodes, storageType); } catch (NotEnoughReplicasException e1) { // find the second replica DatanodeDescriptor newLocal=null; - for(DatanodeDescriptor nextNode : results) { + for(DatanodeStorageInfo resultStorage : results) { + DatanodeDescriptor nextNode = resultStorage.getDatanodeDescriptor(); if (nextNode != localMachine) { newLocal = nextNode; break; @@ -397,16 +415,16 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { if (newLocal != null) { try { return chooseRandom(newLocal.getNetworkLocation(), excludedNodes, - blocksize, maxNodesPerRack, results, avoidStaleNodes); + blocksize, maxNodesPerRack, results, avoidStaleNodes, storageType); } catch(NotEnoughReplicasException e2) { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + maxNodesPerRack, results, avoidStaleNodes, storageType); } } else { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + maxNodesPerRack, results, avoidStaleNodes, storageType); } } } @@ -423,48 +441,51 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { Set excludedNodes, long blocksize, int maxReplicasPerRack, - List results, - boolean avoidStaleNodes) + List results, + boolean avoidStaleNodes, + StorageType storageType) throws NotEnoughReplicasException { int oldNumOfReplicas = results.size(); // randomly choose one node from remote racks try { chooseRandom(numOfReplicas, "~" + localMachine.getNetworkLocation(), excludedNodes, blocksize, maxReplicasPerRack, results, - avoidStaleNodes); + avoidStaleNodes, storageType); } catch (NotEnoughReplicasException e) { chooseRandom(numOfReplicas-(results.size()-oldNumOfReplicas), localMachine.getNetworkLocation(), excludedNodes, blocksize, - maxReplicasPerRack, results, avoidStaleNodes); + maxReplicasPerRack, results, avoidStaleNodes, storageType); } } /** * Randomly choose one target from the given scope. - * @return the chosen node, if there is any. + * @return the chosen storage, if there is any. */ - protected DatanodeDescriptor chooseRandom(String scope, + protected DatanodeStorageInfo chooseRandom(String scope, Set excludedNodes, long blocksize, int maxNodesPerRack, - List results, - boolean avoidStaleNodes) + List results, + boolean avoidStaleNodes, + StorageType storageType) throws NotEnoughReplicasException { return chooseRandom(1, scope, excludedNodes, blocksize, maxNodesPerRack, - results, avoidStaleNodes); + results, avoidStaleNodes, storageType); } /** * Randomly choose numOfReplicas targets from the given scope. * @return the first chosen node, if there is any. */ - protected DatanodeDescriptor chooseRandom(int numOfReplicas, + protected DatanodeStorageInfo chooseRandom(int numOfReplicas, String scope, Set excludedNodes, long blocksize, int maxNodesPerRack, - List results, - boolean avoidStaleNodes) + List results, + boolean avoidStaleNodes, + StorageType storageType) throws NotEnoughReplicasException { int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes( @@ -475,24 +496,28 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { builder.setLength(0); builder.append("["); } - boolean badTarget = false; - DatanodeDescriptor firstChosen = null; + boolean goodTarget = false; + DatanodeStorageInfo firstChosen = null; while(numOfReplicas > 0 && numOfAvailableNodes > 0) { DatanodeDescriptor chosenNode = (DatanodeDescriptor)clusterMap.chooseRandom(scope); if (excludedNodes.add(chosenNode)) { //was not in the excluded list numOfAvailableNodes--; - int newExcludedNodes = addIfIsGoodTarget(chosenNode, excludedNodes, - blocksize, maxNodesPerRack, considerLoad, results, avoidStaleNodes); - if (newExcludedNodes >= 0) { - numOfReplicas--; - if (firstChosen == null) { - firstChosen = chosenNode; + final DatanodeStorageInfo[] storages = DFSUtil.shuffle( + chosenNode.getStorageInfos()); + for(int i = 0; i < storages.length && !goodTarget; i++) { + final int newExcludedNodes = addIfIsGoodTarget(storages[i], + excludedNodes, blocksize, maxNodesPerRack, considerLoad, results, + avoidStaleNodes, storageType); + goodTarget = newExcludedNodes >= 0; + if (goodTarget) { + numOfReplicas--; + if (firstChosen == null) { + firstChosen = storages[i]; + } + numOfAvailableNodes -= newExcludedNodes; } - numOfAvailableNodes -= newExcludedNodes; - } else { - badTarget = true; } } } @@ -500,7 +525,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { if (numOfReplicas>0) { String detail = enableDebugLogging; if (LOG.isDebugEnabled()) { - if (badTarget && builder != null) { + if (!goodTarget && builder != null) { detail = builder.append("]").toString(); builder.setLength(0); } else detail = ""; @@ -512,43 +537,46 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { } /** - * If the given node is a good target, add it to the result list and + * If the given storage is a good target, add it to the result list and * update the set of excluded nodes. * @return -1 if the given is not a good target; * otherwise, return the number of nodes added to excludedNodes set. */ - int addIfIsGoodTarget(DatanodeDescriptor node, + int addIfIsGoodTarget(DatanodeStorageInfo storage, Set excludedNodes, long blockSize, int maxNodesPerRack, boolean considerLoad, - List results, - boolean avoidStaleNodes) { - if (isGoodTarget(node, blockSize, maxNodesPerRack, considerLoad, - results, avoidStaleNodes)) { - results.add(node); + List results, + boolean avoidStaleNodes, + StorageType storageType) { + if (isGoodTarget(storage, blockSize, maxNodesPerRack, considerLoad, + results, avoidStaleNodes, storageType)) { + results.add(storage); // add node and related nodes to excludedNode - return addToExcludedNodes(node, excludedNodes); + return addToExcludedNodes(storage.getDatanodeDescriptor(), excludedNodes); } else { return -1; } } - private static void logNodeIsNotChosen(DatanodeDescriptor node, String reason) { + private static void logNodeIsNotChosen(DatanodeStorageInfo storage, String reason) { if (LOG.isDebugEnabled()) { + final DatanodeDescriptor node = storage.getDatanodeDescriptor(); // build the error message for later use. debugLoggingBuilder.get() .append(node).append(": ") - .append("Node ").append(NodeBase.getPath(node)) + .append("Storage ").append(storage) + .append("at node ").append(NodeBase.getPath(node)) .append(" is not chosen because ") .append(reason); } } /** - * Determine if a node is a good target. + * Determine if a storage is a good target. * - * @param node The target node + * @param storage The target storage * @param blockSize Size of block * @param maxTargetPerRack Maximum number of targets per rack. The value of * this parameter depends on the number of racks in @@ -561,29 +589,47 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { * does not have too much load, * and the rack does not have too many nodes. */ - private boolean isGoodTarget(DatanodeDescriptor node, + private boolean isGoodTarget(DatanodeStorageInfo storage, long blockSize, int maxTargetPerRack, boolean considerLoad, - List results, - boolean avoidStaleNodes) { - // check if the node is (being) decommissed + List results, + boolean avoidStaleNodes, + StorageType storageType) { + if (storage.getStorageType() != storageType) { + logNodeIsNotChosen(storage, + "storage types do not match, where the expected storage type is " + + storageType); + return false; + } + if (storage.getState() == State.READ_ONLY) { + logNodeIsNotChosen(storage, "storage is read-only"); + return false; + } + DatanodeDescriptor node = storage.getDatanodeDescriptor(); + // check if the node is (being) decommissioned if (node.isDecommissionInProgress() || node.isDecommissioned()) { - logNodeIsNotChosen(node, "the node is (being) decommissioned "); + logNodeIsNotChosen(storage, "the node is (being) decommissioned "); return false; } if (avoidStaleNodes) { if (node.isStale(this.staleInterval)) { - logNodeIsNotChosen(node, "the node is stale "); + logNodeIsNotChosen(storage, "the node is stale "); return false; } } + final long requiredSize = blockSize * HdfsConstants.MIN_BLOCKS_FOR_WRITE; + if (requiredSize > storage.getRemaining()) { + logNodeIsNotChosen(storage, "the storage does not have enough space "); + return false; + } + //TODO: move getBlocksScheduled() to DatanodeStorageInfo. long remaining = node.getRemaining() - (node.getBlocksScheduled() * blockSize); // check the remaining capacity of the target machine - if (blockSize* HdfsConstants.MIN_BLOCKS_FOR_WRITE>remaining) { - logNodeIsNotChosen(node, "the node does not have enough space "); + if (requiredSize > remaining) { + logNodeIsNotChosen(storage, "the node does not have enough space "); return false; } @@ -595,7 +641,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { avgLoad = (double)stats.getTotalLoad()/size; } if (node.getXceiverCount() > (2.0 * avgLoad)) { - logNodeIsNotChosen(node, "the node is too busy "); + logNodeIsNotChosen(storage, "the node is too busy "); return false; } } @@ -603,13 +649,14 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { // check if the target rack has chosen too many nodes String rackname = node.getNetworkLocation(); int counter=1; - for(Node result : results) { - if (rackname.equals(result.getNetworkLocation())) { + for(DatanodeStorageInfo resultStorage : results) { + if (rackname.equals( + resultStorage.getDatanodeDescriptor().getNetworkLocation())) { counter++; } } if (counter>maxTargetPerRack) { - logNodeIsNotChosen(node, "the rack has too many chosen nodes "); + logNodeIsNotChosen(storage, "the rack has too many chosen nodes "); return false; } return true; @@ -621,37 +668,40 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { * starts from the writer and traverses all nodes * This is basically a traveling salesman problem. */ - private DatanodeDescriptor[] getPipeline(Node writer, - DatanodeDescriptor[] nodes) { - if (nodes.length==0) return nodes; - + private DatanodeStorageInfo[] getPipeline(Node writer, + DatanodeStorageInfo[] storages) { + if (storages.length == 0) { + return storages; + } + synchronized(clusterMap) { int index=0; if (writer == null || !clusterMap.contains(writer)) { - writer = nodes[0]; + writer = storages[0].getDatanodeDescriptor(); } - for(;indexcurrentDistance) { shortestDistance = currentDistance; - shortestNode = currentNode; + shortestStorage = storages[i]; shortestIndex = i; } } //switch position index & shortestIndex if (index != shortestIndex) { - nodes[shortestIndex] = nodes[index]; - nodes[index] = shortestNode; + storages[shortestIndex] = storages[index]; + storages[index] = shortestStorage; } - writer = shortestNode; + writer = shortestStorage.getDatanodeDescriptor(); } } - return nodes; + return storages; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java index 1b8f916dd2c..f31236b675d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java @@ -25,6 +25,8 @@ import java.util.Map; import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.namenode.FSClusterStats; import org.apache.hadoop.net.NetworkTopology; @@ -64,81 +66,87 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau * @return the chosen node */ @Override - protected DatanodeDescriptor chooseLocalNode(Node localMachine, + protected DatanodeStorageInfo chooseLocalStorage(Node localMachine, Set excludedNodes, long blocksize, int maxNodesPerRack, - List results, boolean avoidStaleNodes) - throws NotEnoughReplicasException { + List results, boolean avoidStaleNodes, + StorageType storageType) throws NotEnoughReplicasException { // if no local machine, randomly choose one node if (localMachine == null) return chooseRandom(NodeBase.ROOT, excludedNodes, - blocksize, maxNodesPerRack, results, avoidStaleNodes); + blocksize, maxNodesPerRack, results, avoidStaleNodes, storageType); + // otherwise try local machine first if (localMachine instanceof DatanodeDescriptor) { DatanodeDescriptor localDataNode = (DatanodeDescriptor)localMachine; - // otherwise try local machine first if (excludedNodes.add(localMachine)) { // was not in the excluded list - if (addIfIsGoodTarget(localDataNode, excludedNodes, blocksize, - maxNodesPerRack, false, results, avoidStaleNodes) >= 0) { - return localDataNode; + for(DatanodeStorageInfo localStorage : DFSUtil.shuffle( + localDataNode.getStorageInfos())) { + if (addIfIsGoodTarget(localStorage, excludedNodes, blocksize, + maxNodesPerRack, false, results, avoidStaleNodes, storageType) >= 0) { + return localStorage; + } } } } // try a node on local node group - DatanodeDescriptor chosenNode = chooseLocalNodeGroup( + DatanodeStorageInfo chosenStorage = chooseLocalNodeGroup( (NetworkTopologyWithNodeGroup)clusterMap, localMachine, excludedNodes, - blocksize, maxNodesPerRack, results, avoidStaleNodes); - if (chosenNode != null) { - return chosenNode; + blocksize, maxNodesPerRack, results, avoidStaleNodes, storageType); + if (chosenStorage != null) { + return chosenStorage; } // try a node on local rack return chooseLocalRack(localMachine, excludedNodes, - blocksize, maxNodesPerRack, results, avoidStaleNodes); + blocksize, maxNodesPerRack, results, avoidStaleNodes, storageType); + } + + /** @return the node of the second replica */ + private static DatanodeDescriptor secondNode(Node localMachine, + List results) { + // find the second replica + for(DatanodeStorageInfo nextStorage : results) { + DatanodeDescriptor nextNode = nextStorage.getDatanodeDescriptor(); + if (nextNode != localMachine) { + return nextNode; + } + } + return null; } - @Override - protected DatanodeDescriptor chooseLocalRack(Node localMachine, + protected DatanodeStorageInfo chooseLocalRack(Node localMachine, Set excludedNodes, long blocksize, int maxNodesPerRack, - List results, boolean avoidStaleNodes) - throws NotEnoughReplicasException { + List results, boolean avoidStaleNodes, + StorageType storageType) throws NotEnoughReplicasException { // no local machine, so choose a random machine if (localMachine == null) { - return chooseRandom(NodeBase.ROOT, excludedNodes, - blocksize, maxNodesPerRack, results, - avoidStaleNodes); + return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, + maxNodesPerRack, results, avoidStaleNodes, storageType); } // choose one from the local rack, but off-nodegroup try { - return chooseRandom(NetworkTopology.getFirstHalf( - localMachine.getNetworkLocation()), - excludedNodes, blocksize, - maxNodesPerRack, results, - avoidStaleNodes); + final String scope = NetworkTopology.getFirstHalf(localMachine.getNetworkLocation()); + return chooseRandom(scope, excludedNodes, blocksize, maxNodesPerRack, + results, avoidStaleNodes, storageType); } catch (NotEnoughReplicasException e1) { // find the second replica - DatanodeDescriptor newLocal=null; - for(DatanodeDescriptor nextNode : results) { - if (nextNode != localMachine) { - newLocal = nextNode; - break; - } - } + final DatanodeDescriptor newLocal = secondNode(localMachine, results); if (newLocal != null) { try { return chooseRandom( clusterMap.getRack(newLocal.getNetworkLocation()), excludedNodes, - blocksize, maxNodesPerRack, results, avoidStaleNodes); + blocksize, maxNodesPerRack, results, avoidStaleNodes, storageType); } catch(NotEnoughReplicasException e2) { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + maxNodesPerRack, results, avoidStaleNodes, storageType); } } else { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + maxNodesPerRack, results, avoidStaleNodes, storageType); } } } @@ -146,8 +154,9 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau @Override protected void chooseRemoteRack(int numOfReplicas, DatanodeDescriptor localMachine, Set excludedNodes, - long blocksize, int maxReplicasPerRack, List results, - boolean avoidStaleNodes) throws NotEnoughReplicasException { + long blocksize, int maxReplicasPerRack, List results, + boolean avoidStaleNodes, StorageType storageType) + throws NotEnoughReplicasException { int oldNumOfReplicas = results.size(); final String rackLocation = NetworkTopology.getFirstHalf( @@ -155,12 +164,12 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau try { // randomly choose from remote racks chooseRandom(numOfReplicas, "~" + rackLocation, excludedNodes, blocksize, - maxReplicasPerRack, results, avoidStaleNodes); + maxReplicasPerRack, results, avoidStaleNodes, storageType); } catch (NotEnoughReplicasException e) { // fall back to the local rack chooseRandom(numOfReplicas - (results.size() - oldNumOfReplicas), rackLocation, excludedNodes, blocksize, - maxReplicasPerRack, results, avoidStaleNodes); + maxReplicasPerRack, results, avoidStaleNodes, storageType); } } @@ -170,46 +179,40 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau * if still no such node is available, choose a random node in the cluster. * @return the chosen node */ - private DatanodeDescriptor chooseLocalNodeGroup( + private DatanodeStorageInfo chooseLocalNodeGroup( NetworkTopologyWithNodeGroup clusterMap, Node localMachine, Set excludedNodes, long blocksize, int maxNodesPerRack, - List results, boolean avoidStaleNodes) - throws NotEnoughReplicasException { + List results, boolean avoidStaleNodes, + StorageType storageType) throws NotEnoughReplicasException { // no local machine, so choose a random machine if (localMachine == null) { - return chooseRandom(NodeBase.ROOT, excludedNodes, - blocksize, maxNodesPerRack, results, avoidStaleNodes); + return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, + maxNodesPerRack, results, avoidStaleNodes, storageType); } // choose one from the local node group try { return chooseRandom( clusterMap.getNodeGroup(localMachine.getNetworkLocation()), - excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes); + excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, + storageType); } catch (NotEnoughReplicasException e1) { - // find the second replica - DatanodeDescriptor newLocal=null; - for(DatanodeDescriptor nextNode : results) { - if (nextNode != localMachine) { - newLocal = nextNode; - break; - } - } + final DatanodeDescriptor newLocal = secondNode(localMachine, results); if (newLocal != null) { try { return chooseRandom( clusterMap.getNodeGroup(newLocal.getNetworkLocation()), excludedNodes, blocksize, maxNodesPerRack, results, - avoidStaleNodes); + avoidStaleNodes, storageType); } catch(NotEnoughReplicasException e2) { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + maxNodesPerRack, results, avoidStaleNodes, storageType); } } else { //otherwise randomly choose one from the network return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes); + maxNodesPerRack, results, avoidStaleNodes, storageType); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index b006001f59a..4cdac91faf6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; @@ -54,9 +55,9 @@ public class DatanodeDescriptor extends DatanodeInfo { @InterfaceStability.Evolving public static class BlockTargetPair { public final Block block; - public final DatanodeDescriptor[] targets; + public final DatanodeStorageInfo[] targets; - BlockTargetPair(Block block, DatanodeDescriptor[] targets) { + BlockTargetPair(Block block, DatanodeStorageInfo[] targets) { this.block = block; this.targets = targets; } @@ -215,14 +216,15 @@ public class DatanodeDescriptor extends DatanodeInfo { return false; } - public DatanodeStorageInfo getStorageInfo(String storageID) { + DatanodeStorageInfo getStorageInfo(String storageID) { synchronized (storageMap) { return storageMap.get(storageID); } } - public Collection getStorageInfos() { + DatanodeStorageInfo[] getStorageInfos() { synchronized (storageMap) { - return new ArrayList(storageMap.values()); + final Collection storages = storageMap.values(); + return storages.toArray(new DatanodeStorageInfo[storages.size()]); } } @@ -254,14 +256,6 @@ public class DatanodeDescriptor extends DatanodeInfo { return false; } - /** - * Used for testing only - * @return the head of the blockList - */ - protected BlockInfo getHead(){ - return getBlockIterator().next(); - } - /** * Replace specified old block with a new one in the DataNodeDescriptor. * @@ -325,20 +319,15 @@ public class DatanodeDescriptor extends DatanodeInfo { } private static class BlockIterator implements Iterator { - private final int maxIndex; private int index = 0; - private List> iterators = new ArrayList>(); + private final List> iterators; - private BlockIterator(final Iterable storages) { + private BlockIterator(final DatanodeStorageInfo... storages) { + List> iterators = new ArrayList>(); for (DatanodeStorageInfo e : storages) { iterators.add(e.getBlockIterator()); } - maxIndex = iterators.size() - 1; - } - - private BlockIterator(final DatanodeStorageInfo storage) { - iterators.add(storage.getBlockIterator()); - maxIndex = iterators.size() - 1; + this.iterators = Collections.unmodifiableList(iterators); } @Override @@ -359,7 +348,7 @@ public class DatanodeDescriptor extends DatanodeInfo { } private void update() { - while(index < maxIndex && !iterators.get(index).hasNext()) { + while(index < iterators.size() - 1 && !iterators.get(index).hasNext()) { index++; } } @@ -375,7 +364,7 @@ public class DatanodeDescriptor extends DatanodeInfo { /** * Store block replication work. */ - void addBlockToBeReplicated(Block block, DatanodeDescriptor[] targets) { + void addBlockToBeReplicated(Block block, DatanodeStorageInfo[] targets) { assert(block != null && targets != null && targets.length > 0); replicateBlocks.offer(new BlockTargetPair(block, targets)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index ba0e142b36c..6695e072446 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -32,7 +32,9 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport; * by this class. */ public class DatanodeStorageInfo { - static DatanodeInfo[] toDatanodeInfos(DatanodeStorageInfo[] storages) { + public static final DatanodeStorageInfo[] EMPTY_ARRAY = {}; + + public static DatanodeInfo[] toDatanodeInfos(DatanodeStorageInfo[] storages) { return toDatanodeInfos(Arrays.asList(storages)); } static DatanodeInfo[] toDatanodeInfos(List storages) { @@ -43,6 +45,22 @@ public class DatanodeStorageInfo { return datanodes; } + public static String[] toStorageIDs(DatanodeStorageInfo[] storages) { + String[] storageIDs = new String[storages.length]; + for(int i = 0; i < storageIDs.length; i++) { + storageIDs[i] = storages[i].getStorageID(); + } + return storageIDs; + } + + public static StorageType[] toStorageTypes(DatanodeStorageInfo[] storages) { + StorageType[] storageTypes = new StorageType[storages.length]; + for(int i = 0; i < storageTypes.length; i++) { + storageTypes[i] = storages[i].getStorageType(); + } + return storageTypes; + } + /** * Iterates over the list of blocks belonging to the data-node. */ @@ -207,6 +225,22 @@ public class DatanodeStorageInfo { return dn; } + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } else if (obj == null || !(obj instanceof DatanodeStorageInfo)) { + return false; + } + final DatanodeStorageInfo that = (DatanodeStorageInfo)obj; + return this.storageID.equals(that.storageID); + } + + @Override + public int hashCode() { + return storageID.hashCode(); + } + @Override public String toString() { return "[" + storageType + "]" + storageID + ":" + state; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java index 6b07b789341..c99d53fc148 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java @@ -76,7 +76,7 @@ class PendingReplicationBlocks { * @param block The corresponding block * @param targets The DataNodes where replicas of the block should be placed */ - void increment(Block block, DatanodeDescriptor[] targets) { + void increment(Block block, DatanodeStorageInfo[] targets) { synchronized (pendingReplications) { PendingBlockInfo found = pendingReplications.get(block); if (found == null) { @@ -95,14 +95,14 @@ class PendingReplicationBlocks { * * @param The DataNode that finishes the replication */ - void decrement(Block block, DatanodeDescriptor dn) { + void decrement(Block block, DatanodeDescriptor dn, String storageID) { synchronized (pendingReplications) { PendingBlockInfo found = pendingReplications.get(block); if (found != null) { if(LOG.isDebugEnabled()) { LOG.debug("Removing pending replication for " + block); } - found.decrementReplicas(dn); + found.decrementReplicas(dn.getStorageInfo(storageID)); if (found.getNumReplicas() <= 0) { pendingReplications.remove(block); } @@ -174,12 +174,12 @@ class PendingReplicationBlocks { */ static class PendingBlockInfo { private long timeStamp; - private final List targets; + private final List targets; - PendingBlockInfo(DatanodeDescriptor[] targets) { + PendingBlockInfo(DatanodeStorageInfo[] targets) { this.timeStamp = now(); - this.targets = targets == null ? new ArrayList() - : new ArrayList(Arrays.asList(targets)); + this.targets = targets == null ? new ArrayList() + : new ArrayList(Arrays.asList(targets)); } long getTimeStamp() { @@ -190,16 +190,16 @@ class PendingReplicationBlocks { timeStamp = now(); } - void incrementReplicas(DatanodeDescriptor... newTargets) { + void incrementReplicas(DatanodeStorageInfo... newTargets) { if (newTargets != null) { - for (DatanodeDescriptor dn : newTargets) { + for (DatanodeStorageInfo dn : newTargets) { targets.add(dn); } } } - void decrementReplicas(DatanodeDescriptor dn) { - targets.remove(dn); + void decrementReplicas(DatanodeStorageInfo storage) { + targets.remove(storage); } int getNumReplicas() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 2168022289e..9798eae18b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -141,6 +141,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ClientProtocol; @@ -2495,14 +2496,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } // choose targets for the new block to be allocated. - // TODO: chooseTarget(..) should be changed to return DatanodeStorageInfo's - final DatanodeDescriptor chosenDatanodes[] = getBlockManager().chooseTarget( + final DatanodeStorageInfo targets[] = getBlockManager().chooseTarget( src, replication, clientNode, excludedNodes, blockSize, favoredNodes); - final DatanodeStorageInfo[] targets = new DatanodeStorageInfo[chosenDatanodes.length]; - for(int i = 0; i < targets.length; i++) { - final DatanodeDescriptor dd = chosenDatanodes[i]; - targets[i] = dd.getStorageInfos().iterator().next(); - } // Part II. // Allocate a new block, add it to the INode and the BlocksMap. @@ -2644,7 +2639,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, LocatedBlock makeLocatedBlock(Block blk, DatanodeStorageInfo[] locs, long offset) throws IOException { - LocatedBlock lBlk = LocatedBlock.createLocatedBlock( + LocatedBlock lBlk = new LocatedBlock( getExtendedBlock(blk), locs, offset, false); getBlockManager().setBlockToken( lBlk, BlockTokenSecretManager.AccessMode.WRITE); @@ -2653,7 +2648,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, /** @see NameNode#getAdditionalDatanode(String, ExtendedBlock, DatanodeInfo[], DatanodeInfo[], int, String) */ LocatedBlock getAdditionalDatanode(String src, final ExtendedBlock blk, - final DatanodeInfo[] existings, final Set excludes, + final DatanodeInfo[] existings, final String[] storageIDs, + final Set excludes, final int numAdditionalNodes, final String clientName ) throws IOException { //check if the feature is enabled @@ -2661,7 +2657,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, final DatanodeDescriptor clientnode; final long preferredblocksize; - final List chosen; + final List chosen; checkOperation(OperationCategory.READ); byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); readLock(); @@ -2679,23 +2675,18 @@ public class FSNamesystem implements Namesystem, FSClusterStats, clientnode = file.getClientNode(); preferredblocksize = file.getPreferredBlockSize(); - //find datanode descriptors - chosen = new ArrayList(); - for(DatanodeInfo d : existings) { - final DatanodeDescriptor descriptor = blockManager.getDatanodeManager( - ).getDatanode(d); - if (descriptor != null) { - chosen.add(descriptor); - } - } + //find datanode storages + final DatanodeManager dm = blockManager.getDatanodeManager(); + chosen = Arrays.asList(dm.getDatanodeStorageInfos(existings, storageIDs)); } finally { readUnlock(); } // choose new datanodes. - final DatanodeInfo[] targets = blockManager.getBlockPlacementPolicy( + final DatanodeStorageInfo[] targets = blockManager.getBlockPlacementPolicy( ).chooseTarget(src, numAdditionalNodes, clientnode, chosen, true, - excludes, preferredblocksize); + // TODO: get storage type from the file + excludes, preferredblocksize, StorageType.DEFAULT); final LocatedBlock lb = new LocatedBlock(blk, targets); blockManager.setBlockToken(lb, AccessMode.COPY); return lb; @@ -5634,9 +5625,9 @@ public class FSNamesystem implements Namesystem, FSClusterStats, for (int i = 0; i < blocks.length; i++) { ExtendedBlock blk = blocks[i].getBlock(); DatanodeInfo[] nodes = blocks[i].getLocations(); + String[] storageIDs = blocks[i].getStorageIDs(); for (int j = 0; j < nodes.length; j++) { - //TODO: add "storageID to LocatedBlock - blockManager.findAndMarkBlockAsCorrupt(blk, nodes[j], "STORAGE_ID", + blockManager.findAndMarkBlockAsCorrupt(blk, nodes[j], storageIDs[j], "client machine reported it"); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 45fbd3b0e14..4c2b5af68b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -566,7 +566,8 @@ class NameNodeRpcServer implements NamenodeProtocols { @Override // ClientProtocol public LocatedBlock getAdditionalDatanode(final String src, final ExtendedBlock blk, - final DatanodeInfo[] existings, final DatanodeInfo[] excludes, + final DatanodeInfo[] existings, final String[] existingStorageIDs, + final DatanodeInfo[] excludes, final int numAdditionalNodes, final String clientName ) throws IOException { if (LOG.isDebugEnabled()) { @@ -587,8 +588,8 @@ class NameNodeRpcServer implements NamenodeProtocols { excludeSet.add(node); } } - return namesystem.getAdditionalDatanode(src, blk, - existings, excludeSet, numAdditionalNodes, clientName); + return namesystem.getAdditionalDatanode(src, blk, existings, + existingStorageIDs, excludeSet, numAdditionalNodes, clientName); } /** * The client needs to give up on the block. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 288455bd2b3..d84009f1d7d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -53,6 +53,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Options; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -61,6 +62,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; @@ -158,7 +160,7 @@ public class NamenodeWebHdfsMethods { static DatanodeInfo chooseDatanode(final NameNode namenode, final String path, final HttpOpParam.Op op, final long openOffset, - final long blocksize, Configuration conf) throws IOException { + final long blocksize, final Configuration conf) throws IOException { final BlockManager bm = namenode.getNamesystem().getBlockManager(); if (op == PutOpParam.Op.CREATE) { @@ -166,11 +168,13 @@ public class NamenodeWebHdfsMethods { final DatanodeDescriptor clientNode = bm.getDatanodeManager( ).getDatanodeByHost(getRemoteAddress()); if (clientNode != null) { - final DatanodeDescriptor[] datanodes = bm.getBlockPlacementPolicy() + final DatanodeStorageInfo[] storages = bm.getBlockPlacementPolicy() .chooseTarget(path, 1, clientNode, - new ArrayList(), false, null, blocksize); - if (datanodes.length > 0) { - return datanodes[0]; + new ArrayList(), false, null, blocksize, + // TODO: get storage type from the file + StorageType.DEFAULT); + if (storages.length > 0) { + return storages[0].getDatanodeDescriptor(); } } } else if (op == GetOpParam.Op.OPEN diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java index 7a58c6162fe..1798d664f93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java @@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; /**************************************************** * A BlockCommand is an instruction to a datanode @@ -46,9 +47,10 @@ public class BlockCommand extends DatanodeCommand { */ public static final long NO_ACK = Long.MAX_VALUE; - String poolId; - Block blocks[]; - DatanodeInfo targets[][]; + final String poolId; + final Block[] blocks; + final DatanodeInfo[][] targets; + final String[][] targetStorageIDs; /** * Create BlockCommand for transferring blocks to another datanode @@ -60,21 +62,26 @@ public class BlockCommand extends DatanodeCommand { this.poolId = poolId; blocks = new Block[blocktargetlist.size()]; targets = new DatanodeInfo[blocks.length][]; + targetStorageIDs = new String[blocks.length][]; + for(int i = 0; i < blocks.length; i++) { BlockTargetPair p = blocktargetlist.get(i); blocks[i] = p.block; - targets[i] = p.targets; + targets[i] = DatanodeStorageInfo.toDatanodeInfos(p.targets); + targetStorageIDs[i] = DatanodeStorageInfo.toStorageIDs(p.targets); } } - private static final DatanodeInfo[][] EMPTY_TARGET = {}; + private static final DatanodeInfo[][] EMPTY_TARGET_DATANODES = {}; + private static final String[][] EMPTY_TARGET_STORAGEIDS = {}; /** * Create BlockCommand for the given action * @param blocks blocks related to the action */ public BlockCommand(int action, String poolId, Block blocks[]) { - this(action, poolId, blocks, EMPTY_TARGET); + this(action, poolId, blocks, EMPTY_TARGET_DATANODES, + EMPTY_TARGET_STORAGEIDS); } /** @@ -82,11 +89,12 @@ public class BlockCommand extends DatanodeCommand { * @param blocks blocks related to the action */ public BlockCommand(int action, String poolId, Block[] blocks, - DatanodeInfo[][] targets) { + DatanodeInfo[][] targets, String[][] targetStorageIDs) { super(action); this.poolId = poolId; this.blocks = blocks; this.targets = targets; + this.targetStorageIDs = targetStorageIDs; } public String getBlockPoolId() { @@ -100,4 +108,8 @@ public class BlockCommand extends DatanodeCommand { public DatanodeInfo[][] getTargets() { return targets; } + + public String[][] getTargetStorageIDs() { + return targetStorageIDs; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index 3f09aeeb3c7..968b0740355 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -141,6 +141,7 @@ message GetAdditionalDatanodeRequestProto { repeated DatanodeInfoProto excludes = 4; required uint32 numAdditionalNodes = 5; required string clientName = 6; + repeated string existingStorageIDs = 7; } message GetAdditionalDatanodeResponseProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto index fe9ddce7c36..2f845e08a43 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto @@ -105,10 +105,12 @@ message BlockCommandProto { INVALIDATE = 2; // Invalidate blocks SHUTDOWN = 3; // Shutdown the datanode } + required Action action = 1; required string blockPoolId = 2; repeated BlockProto blocks = 3; repeated DatanodeInfosProto targets = 4; + repeated StorageIDsProto targetStorageIDs = 5; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index da0b8322aa6..22121a73b5b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -121,6 +121,13 @@ enum StorageTypeProto { SSD = 2; } +/** + * A list of storage IDs. + */ +message StorageIDsProto { + repeated string storageIDs = 1; +} + /** * A LocatedBlock gives information about a block and its location. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 696a320e23c..123ebec77ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -860,9 +860,35 @@ public class DFSTestUtil { public static DatanodeStorageInfo createDatanodeStorageInfo( String storageID, String ip) { + return createDatanodeStorageInfo(storageID, ip, "defaultRack"); + } + public static DatanodeStorageInfo[] createDatanodeStorageInfos(String[] racks) { + return createDatanodeStorageInfos(racks.length, racks); + } + public static DatanodeStorageInfo[] createDatanodeStorageInfos(int n, String... racks) { + DatanodeStorageInfo[] storages = new DatanodeStorageInfo[n]; + for(int i = storages.length; i > 0; ) { + final String storageID = "s" + i; + final String ip = i + "." + i + "." + i + "." + i; + i--; + final String rack = i < racks.length? racks[i]: "defaultRack"; + storages[i] = createDatanodeStorageInfo(storageID, ip, rack); + } + return storages; + } + public static DatanodeStorageInfo createDatanodeStorageInfo( + String storageID, String ip, String rack) { + final DatanodeStorage storage = new DatanodeStorage(storageID); return new DatanodeStorageInfo( - getDatanodeDescriptor(ip, "defaultRack"), - new DatanodeStorage(storageID)); + BlockManagerTestUtil.getDatanodeDescriptor(ip, rack, storage), storage); + } + public static DatanodeDescriptor[] toDatanodeDescriptor( + DatanodeStorageInfo[] storages) { + DatanodeDescriptor[] datanodes = new DatanodeDescriptor[storages.length]; + for(int i = 0; i < datanodes.length; i++) { + datanodes[i] = storages[i].getDatanodeDescriptor(); + } + return datanodes; } public static DatanodeDescriptor getDatanodeDescriptor(String ipAddr, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index 05bb5e977a4..d8d4329d1b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -539,8 +539,9 @@ public class TestPBHelper { dnInfos[0][0] = DFSTestUtil.getLocalDatanodeInfo(); dnInfos[1][0] = DFSTestUtil.getLocalDatanodeInfo(); dnInfos[1][1] = DFSTestUtil.getLocalDatanodeInfo(); + String[][] storageIDs = {{"s00"}, {"s10", "s11"}}; BlockCommand bc = new BlockCommand(DatanodeProtocol.DNA_TRANSFER, "bp1", - blocks, dnInfos); + blocks, dnInfos, storageIDs); BlockCommandProto bcProto = PBHelper.convert(bc); BlockCommand bc2 = PBHelper.convert(bcProto); assertEquals(bc.getAction(), bc2.getAction()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java index 243a2917f8a..8740b8471d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java @@ -227,10 +227,16 @@ public class BlockManagerTestUtil { public static DatanodeDescriptor getDatanodeDescriptor(String ipAddr, String rackLocation, boolean initializeStorage) { + return getDatanodeDescriptor(ipAddr, rackLocation, + initializeStorage? new DatanodeStorage(DatanodeStorage.newStorageID()): null); + } + + public static DatanodeDescriptor getDatanodeDescriptor(String ipAddr, + String rackLocation, DatanodeStorage storage) { DatanodeDescriptor dn = DFSTestUtil.getDatanodeDescriptor(ipAddr, DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT, rackLocation); - if (initializeStorage) { - dn.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); + if (storage != null) { + dn.updateStorage(storage); } return dn; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 2eaf9a0b128..f1b2cf77fef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -22,10 +22,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import java.io.IOException; import java.util.ArrayList; -import java.util.Iterator; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.Map.Entry; @@ -45,7 +49,6 @@ import org.apache.hadoop.net.NetworkTopology; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -import static org.mockito.Mockito.*; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; @@ -53,6 +56,7 @@ import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.Lists; public class TestBlockManager { + private DatanodeStorageInfo[] storages; private List nodes; private List rackA; private List rackB; @@ -81,14 +85,15 @@ public class TestBlockManager { fsn = Mockito.mock(FSNamesystem.class); Mockito.doReturn(true).when(fsn).hasWriteLock(); bm = new BlockManager(fsn, fsn, conf); - nodes = ImmutableList.of( - BlockManagerTestUtil.getDatanodeDescriptor("1.1.1.1", "/rackA", true), - BlockManagerTestUtil.getDatanodeDescriptor("2.2.2.2", "/rackA", true), - BlockManagerTestUtil.getDatanodeDescriptor("3.3.3.3", "/rackA", true), - BlockManagerTestUtil.getDatanodeDescriptor("4.4.4.4", "/rackB", true), - BlockManagerTestUtil.getDatanodeDescriptor("5.5.5.5", "/rackB", true), - BlockManagerTestUtil.getDatanodeDescriptor("6.6.6.6", "/rackB", true) - ); + final String[] racks = { + "/rackA", + "/rackA", + "/rackA", + "/rackB", + "/rackB", + "/rackB"}; + storages = DFSTestUtil.createDatanodeStorageInfos(racks); + nodes = Arrays.asList(DFSTestUtil.toDatanodeDescriptor(storages)); rackA = nodes.subList(0, 3); rackB = nodes.subList(3, 6); } @@ -125,17 +130,18 @@ public class TestBlockManager { } private void doBasicTest(int testIndex) { - List origNodes = getNodes(0, 1); + List origStorages = getStorages(0, 1); + List origNodes = getNodes(origStorages); BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes); - DatanodeDescriptor[] pipeline = scheduleSingleReplication(blockInfo); + DatanodeStorageInfo[] pipeline = scheduleSingleReplication(blockInfo); assertEquals(2, pipeline.length); assertTrue("Source of replication should be one of the nodes the block " + "was on. Was: " + pipeline[0], - origNodes.contains(pipeline[0])); + origStorages.contains(pipeline[0])); assertTrue("Destination of replication should be on the other rack. " + "Was: " + pipeline[1], - rackB.contains(pipeline[1])); + rackB.contains(pipeline[1].getDatanodeDescriptor())); } @@ -156,21 +162,22 @@ public class TestBlockManager { private void doTestTwoOfThreeNodesDecommissioned(int testIndex) throws Exception { // Block originally on A1, A2, B1 - List origNodes = getNodes(0, 1, 3); + List origStorages = getStorages(0, 1, 3); + List origNodes = getNodes(origStorages); BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes); // Decommission two of the nodes (A1, A2) List decomNodes = startDecommission(0, 1); - DatanodeDescriptor[] pipeline = scheduleSingleReplication(blockInfo); + DatanodeStorageInfo[] pipeline = scheduleSingleReplication(blockInfo); assertTrue("Source of replication should be one of the nodes the block " + "was on. Was: " + pipeline[0], - origNodes.contains(pipeline[0])); + origStorages.contains(pipeline[0])); assertEquals("Should have three targets", 3, pipeline.length); boolean foundOneOnRackA = false; for (int i = 1; i < pipeline.length; i++) { - DatanodeDescriptor target = pipeline[i]; + DatanodeDescriptor target = pipeline[i].getDatanodeDescriptor(); if (rackA.contains(target)) { foundOneOnRackA = true; } @@ -199,22 +206,23 @@ public class TestBlockManager { private void doTestAllNodesHoldingReplicasDecommissioned(int testIndex) throws Exception { // Block originally on A1, A2, B1 - List origNodes = getNodes(0, 1, 3); + List origStorages = getStorages(0, 1, 3); + List origNodes = getNodes(origStorages); BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes); // Decommission all of the nodes List decomNodes = startDecommission(0, 1, 3); - DatanodeDescriptor[] pipeline = scheduleSingleReplication(blockInfo); + DatanodeStorageInfo[] pipeline = scheduleSingleReplication(blockInfo); assertTrue("Source of replication should be one of the nodes the block " + "was on. Was: " + pipeline[0], - origNodes.contains(pipeline[0])); + origStorages.contains(pipeline[0])); assertEquals("Should have three targets", 4, pipeline.length); boolean foundOneOnRackA = false; boolean foundOneOnRackB = false; for (int i = 1; i < pipeline.length; i++) { - DatanodeDescriptor target = pipeline[i]; + DatanodeDescriptor target = pipeline[i].getDatanodeDescriptor(); if (rackA.contains(target)) { foundOneOnRackA = true; } else if (rackB.contains(target)) { @@ -251,21 +259,22 @@ public class TestBlockManager { private void doTestOneOfTwoRacksDecommissioned(int testIndex) throws Exception { // Block originally on A1, A2, B1 - List origNodes = getNodes(0, 1, 3); + List origStorages = getStorages(0, 1, 3); + List origNodes = getNodes(origStorages); BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes); // Decommission all of the nodes in rack A List decomNodes = startDecommission(0, 1, 2); - DatanodeDescriptor[] pipeline = scheduleSingleReplication(blockInfo); + DatanodeStorageInfo[] pipeline = scheduleSingleReplication(blockInfo); assertTrue("Source of replication should be one of the nodes the block " + "was on. Was: " + pipeline[0], - origNodes.contains(pipeline[0])); + origStorages.contains(pipeline[0])); assertEquals("Should have three targets", 3, pipeline.length); boolean foundOneOnRackB = false; for (int i = 1; i < pipeline.length; i++) { - DatanodeDescriptor target = pipeline[i]; + DatanodeDescriptor target = pipeline[i].getDatanodeDescriptor(); if (rackB.contains(target)) { foundOneOnRackB = true; } @@ -287,9 +296,9 @@ public class TestBlockManager { rackCNode.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); addNodes(ImmutableList.of(rackCNode)); try { - DatanodeDescriptor[] pipeline2 = scheduleSingleReplication(blockInfo); + DatanodeStorageInfo[] pipeline2 = scheduleSingleReplication(blockInfo); assertEquals(2, pipeline2.length); - assertEquals(rackCNode, pipeline2[1]); + assertEquals(rackCNode, pipeline2[1].getDatanodeDescriptor()); } finally { removeNode(rackCNode); } @@ -311,15 +320,15 @@ public class TestBlockManager { // Originally on only nodes in rack A. List origNodes = rackA; BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes); - DatanodeDescriptor pipeline[] = scheduleSingleReplication(blockInfo); + DatanodeStorageInfo pipeline[] = scheduleSingleReplication(blockInfo); assertEquals(2, pipeline.length); // single new copy assertTrue("Source of replication should be one of the nodes the block " + "was on. Was: " + pipeline[0], - origNodes.contains(pipeline[0])); + origNodes.contains(pipeline[0].getDatanodeDescriptor())); assertTrue("Destination of replication should be on the other rack. " + "Was: " + pipeline[1], - rackB.contains(pipeline[1])); + rackB.contains(pipeline[1].getDatanodeDescriptor())); } @Test @@ -354,19 +363,11 @@ public class TestBlockManager { * pipeline. */ private void fulfillPipeline(BlockInfo blockInfo, - DatanodeDescriptor[] pipeline) throws IOException { + DatanodeStorageInfo[] pipeline) throws IOException { for (int i = 1; i < pipeline.length; i++) { - DatanodeDescriptor dn = pipeline[i]; - - Iterator iterator = dn.getStorageInfos().iterator(); - if (iterator.hasNext()) { - DatanodeStorageInfo storage = iterator.next(); - bm.addBlock(dn, storage.getStorageID(), blockInfo, null); - blockInfo.addStorage(storage); - } else { - throw new RuntimeException("Storage info on node: " + dn.getHostName() - + " is invalid."); - } + DatanodeStorageInfo storage = pipeline[i]; + bm.addBlock(storage.getDatanodeDescriptor(), storage.getStorageID(), blockInfo, null); + blockInfo.addStorage(storage); } } @@ -389,6 +390,22 @@ public class TestBlockManager { } return ret; } + + private List getNodes(List storages) { + List ret = Lists.newArrayList(); + for (DatanodeStorageInfo s : storages) { + ret.add(s.getDatanodeDescriptor()); + } + return ret; + } + + private List getStorages(int ... indexes) { + List ret = Lists.newArrayList(); + for (int idx : indexes) { + ret.add(storages[idx]); + } + return ret; + } private List startDecommission(int ... indexes) { List nodes = getNodes(indexes); @@ -407,7 +424,7 @@ public class TestBlockManager { return blockInfo; } - private DatanodeDescriptor[] scheduleSingleReplication(Block block) { + private DatanodeStorageInfo[] scheduleSingleReplication(Block block) { // list for priority 1 List list_p1 = new ArrayList(); list_p1.add(block); @@ -425,27 +442,29 @@ public class TestBlockManager { assertTrue("replication is pending after work is computed", bm.pendingReplications.getNumReplicas(block) > 0); - LinkedListMultimap repls = getAllPendingReplications(); + LinkedListMultimap repls = getAllPendingReplications(); assertEquals(1, repls.size()); - Entry repl = + Entry repl = repls.entries().iterator().next(); - DatanodeDescriptor[] targets = repl.getValue().targets; + DatanodeStorageInfo[] targets = repl.getValue().targets; - DatanodeDescriptor[] pipeline = new DatanodeDescriptor[1 + targets.length]; + DatanodeStorageInfo[] pipeline = new DatanodeStorageInfo[1 + targets.length]; pipeline[0] = repl.getKey(); System.arraycopy(targets, 0, pipeline, 1, targets.length); return pipeline; } - private LinkedListMultimap getAllPendingReplications() { - LinkedListMultimap repls = + private LinkedListMultimap getAllPendingReplications() { + LinkedListMultimap repls = LinkedListMultimap.create(); for (DatanodeDescriptor dn : nodes) { List thisRepls = dn.getReplicationCommand(10); if (thisRepls != null) { - repls.putAll(dn, thisRepls); + for(DatanodeStorageInfo storage : dn.getStorageInfos()) { + repls.putAll(storage, thisRepls); + } } } return repls; @@ -468,7 +487,7 @@ public class TestBlockManager { addBlockOnNodes(blockId,origNodes.subList(0,1)); List cntNodes = new LinkedList(); - List liveNodes = new LinkedList(); + List liveNodes = new LinkedList(); assertNotNull("Chooses source node for a highest-priority replication" + " even if all available source nodes have reached their replication" @@ -491,7 +510,7 @@ public class TestBlockManager { UnderReplicatedBlocks.QUEUE_VERY_UNDER_REPLICATED)); // Increase the replication count to test replication count > hard limit - DatanodeDescriptor targets[] = { origNodes.get(1) }; + DatanodeStorageInfo targets[] = { origNodes.get(1).getStorageInfos()[0] }; origNodes.get(0).addBlockToBeReplicated(aBlock, targets); assertNull("Does not choose a source node for a highest-priority" @@ -507,8 +526,7 @@ public class TestBlockManager { @Test public void testSafeModeIBR() throws Exception { DatanodeDescriptor node = spy(nodes.get(0)); - Iterator i = node.getStorageInfos().iterator(); - DatanodeStorageInfo ds = i.next(); + DatanodeStorageInfo ds = node.getStorageInfos()[0]; node.setStorageID(ds.getStorageID()); node.isAlive = true; @@ -552,8 +570,7 @@ public class TestBlockManager { @Test public void testSafeModeIBRAfterIncremental() throws Exception { DatanodeDescriptor node = spy(nodes.get(0)); - Iterator i = node.getStorageInfos().iterator(); - DatanodeStorageInfo ds = i.next(); + DatanodeStorageInfo ds = node.getStorageInfos()[0]; node.setStorageID(ds.getStorageID()); node.isAlive = true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java index bf240d7d46d..12674eb318a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java @@ -22,7 +22,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.util.ArrayList; -import java.util.Iterator; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.protocol.Block; @@ -60,9 +59,9 @@ public class TestDatanodeDescriptor { assertEquals(0, dd.numBlocks()); BlockInfo blk = new BlockInfo(new Block(1L), 1); BlockInfo blk1 = new BlockInfo(new Block(2L), 2); - Iterator iterator = dd.getStorageInfos().iterator(); - assertTrue(iterator.hasNext()); - final String storageID = iterator.next().getStorageID(); + DatanodeStorageInfo[] storages = dd.getStorageInfos(); + assertTrue(storages.length > 0); + final String storageID = storages[0].getStorageID(); // add first block assertTrue(dd.addBlock(storageID, blk)); assertEquals(1, dd.numBlocks()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java index fc5ce60b4d9..a5956db8a0c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.junit.Test; /** @@ -62,6 +63,8 @@ public class TestHeartbeatHandling { final DatanodeRegistration nodeReg = DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId); final DatanodeDescriptor dd = NameNodeAdapter.getDatanode(namesystem, nodeReg); + final String storageID = DatanodeStorage.newStorageID(); + dd.updateStorage(new DatanodeStorage(storageID)); final int REMAINING_BLOCKS = 1; final int MAX_REPLICATE_LIMIT = @@ -69,7 +72,7 @@ public class TestHeartbeatHandling { final int MAX_INVALIDATE_LIMIT = DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT; final int MAX_INVALIDATE_BLOCKS = 2*MAX_INVALIDATE_LIMIT+REMAINING_BLOCKS; final int MAX_REPLICATE_BLOCKS = 2*MAX_REPLICATE_LIMIT+REMAINING_BLOCKS; - final DatanodeDescriptor[] ONE_TARGET = new DatanodeDescriptor[1]; + final DatanodeStorageInfo[] ONE_TARGET = {dd.getStorageInfo(storageID)}; try { namesystem.writeLock(); @@ -143,12 +146,15 @@ public class TestHeartbeatHandling { final DatanodeRegistration nodeReg1 = DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId); final DatanodeDescriptor dd1 = NameNodeAdapter.getDatanode(namesystem, nodeReg1); + dd1.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); final DatanodeRegistration nodeReg2 = DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(1), poolId); final DatanodeDescriptor dd2 = NameNodeAdapter.getDatanode(namesystem, nodeReg2); + dd2.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); final DatanodeRegistration nodeReg3 = DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(2), poolId); final DatanodeDescriptor dd3 = NameNodeAdapter.getDatanode(namesystem, nodeReg3); + dd3.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); try { namesystem.writeLock(); @@ -162,9 +168,9 @@ public class TestHeartbeatHandling { dd2.setLastUpdate(System.currentTimeMillis()); dd3.setLastUpdate(System.currentTimeMillis()); final DatanodeStorageInfo[] storages = { - dd1.getStorageInfos().iterator().next(), - dd2.getStorageInfos().iterator().next(), - dd3.getStorageInfos().iterator().next()}; + dd1.getStorageInfos()[0], + dd2.getStorageInfos()[0], + dd3.getStorageInfos()[0]}; BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction( new Block(0, 0, GenerationStamp.LAST_RESERVED_STAMP), 3, BlockUCState.UNDER_RECOVERY, storages); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java index 8eb9ced9e2f..74aef6d6a71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java @@ -43,8 +43,6 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStat import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.junit.Test; -import com.google.common.base.Preconditions; - /** * This class tests the internals of PendingReplicationBlocks.java, * as well as how PendingReplicationBlocks acts in BlockManager @@ -54,22 +52,7 @@ public class TestPendingReplication { private static final int DFS_REPLICATION_INTERVAL = 1; // Number of datanodes in the cluster private static final int DATANODE_COUNT = 5; - - private DatanodeDescriptor genDatanodeId(int seed) { - seed = seed % 256; - String ip = seed + "." + seed + "." + seed + "." + seed; - return DFSTestUtil.getDatanodeDescriptor(ip, null); - } - private DatanodeDescriptor[] genDatanodes(int number) { - Preconditions.checkArgument(number >= 0); - DatanodeDescriptor[] nodes = new DatanodeDescriptor[number]; - for (int i = 0; i < number; i++) { - nodes[i] = genDatanodeId(i); - } - return nodes; - } - @Test public void testPendingReplication() { PendingReplicationBlocks pendingReplications; @@ -79,9 +62,12 @@ public class TestPendingReplication { // // Add 10 blocks to pendingReplications. // - for (int i = 0; i < 10; i++) { + DatanodeStorageInfo[] storages = DFSTestUtil.createDatanodeStorageInfos(10); + for (int i = 0; i < storages.length; i++) { Block block = new Block(i, i, 0); - pendingReplications.increment(block, genDatanodes(i)); + DatanodeStorageInfo[] targets = new DatanodeStorageInfo[i]; + System.arraycopy(storages, 0, targets, 0, i); + pendingReplications.increment(block, targets); } assertEquals("Size of pendingReplications ", 10, pendingReplications.size()); @@ -91,16 +77,18 @@ public class TestPendingReplication { // remove one item and reinsert it // Block blk = new Block(8, 8, 0); - pendingReplications.decrement(blk, genDatanodeId(7)); // removes one replica + pendingReplications.decrement(blk, storages[7].getDatanodeDescriptor(), + storages[7].getStorageID()); // removes one replica assertEquals("pendingReplications.getNumReplicas ", 7, pendingReplications.getNumReplicas(blk)); for (int i = 0; i < 7; i++) { // removes all replicas - pendingReplications.decrement(blk, genDatanodeId(i)); + pendingReplications.decrement(blk, storages[i].getDatanodeDescriptor(), + storages[i].getStorageID()); } assertTrue(pendingReplications.size() == 9); - pendingReplications.increment(blk, genDatanodes(8)); + pendingReplications.increment(blk, DFSTestUtil.createDatanodeStorageInfos(8)); assertTrue(pendingReplications.size() == 10); // @@ -128,7 +116,7 @@ public class TestPendingReplication { for (int i = 10; i < 15; i++) { Block block = new Block(i, i, 0); - pendingReplications.increment(block, genDatanodes(i)); + pendingReplications.increment(block, DFSTestUtil.createDatanodeStorageInfos(i)); } assertTrue(pendingReplications.size() == 15); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index ca1de4fcfa8..21dd5f04262 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; + import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -35,6 +36,7 @@ import java.util.Map; import java.util.Random; import java.util.Set; +import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; @@ -44,6 +46,7 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.LogVerificationAppender; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; @@ -67,6 +70,10 @@ import org.junit.Test; import org.junit.rules.ExpectedException; public class TestReplicationPolicy { + { + ((Log4JLogger)BlockPlacementPolicy.LOG).getLogger().setLevel(Level.ALL); + } + private Random random = DFSUtil.getRandom(); private static final int BLOCK_SIZE = 1024; private static final int NUM_OF_DATANODES = 6; @@ -75,7 +82,7 @@ public class TestReplicationPolicy { private static BlockPlacementPolicy replicator; private static final String filename = "/dummyfile.txt"; private static DatanodeDescriptor dataNodes[]; - private static String[] storageIDs; + private static DatanodeStorageInfo[] storages; // The interval for marking a datanode as stale, private static long staleInterval = DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_DEFAULT; @@ -86,14 +93,15 @@ public class TestReplicationPolicy { @BeforeClass public static void setupCluster() throws Exception { Configuration conf = new HdfsConfiguration(); - dataNodes = new DatanodeDescriptor[] { - DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1"), - DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1"), - DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1/r2"), - DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/d1/r2"), - DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/d2/r3"), - DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d2/r3") - }; + final String[] racks = { + "/d1/r1", + "/d1/r1", + "/d1/r2", + "/d1/r2", + "/d2/r3", + "/d2/r3"}; + storages = DFSTestUtil.createDatanodeStorageInfos(racks); + dataNodes = DFSTestUtil.toDatanodeDescriptor(storages); FileSystem.setDefaultUri(conf, "hdfs://localhost:0"); conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); @@ -124,6 +132,13 @@ public class TestReplicationPolicy { } } + private static boolean isOnSameRack(DatanodeStorageInfo left, DatanodeStorageInfo right) { + return isOnSameRack(left, right.getDatanodeDescriptor()); + } + + private static boolean isOnSameRack(DatanodeStorageInfo left, DatanodeDescriptor right) { + return cluster.isOnSameRack(left.getDatanodeDescriptor(), right); + } /** * In this testcase, client is dataNodes[0]. So the 1st replica should be * placed on dataNodes[0], the 2nd replica should be placed on @@ -139,69 +154,69 @@ public class TestReplicationPolicy { 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 4, 0); // overloaded - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0); assertEquals(targets.length, 0); targets = chooseTarget(1); assertEquals(targets.length, 1); - assertEquals(targets[0], dataNodes[0]); + assertEquals(storages[0], targets[0]); targets = chooseTarget(2); assertEquals(targets.length, 2); - assertEquals(targets[0], dataNodes[0]); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertEquals(storages[0], targets[0]); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3); assertEquals(targets.length, 3); - assertEquals(targets[0], dataNodes[0]); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - assertTrue(cluster.isOnSameRack(targets[1], targets[2])); + assertEquals(storages[0], targets[0]); + assertFalse(isOnSameRack(targets[0], targets[1])); + assertTrue(isOnSameRack(targets[1], targets[2])); targets = chooseTarget(4); assertEquals(targets.length, 4); - assertEquals(targets[0], dataNodes[0]); - assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || - cluster.isOnSameRack(targets[2], targets[3])); - assertFalse(cluster.isOnSameRack(targets[0], targets[2])); + assertEquals(storages[0], targets[0]); + assertTrue(isOnSameRack(targets[1], targets[2]) || + isOnSameRack(targets[2], targets[3])); + assertFalse(isOnSameRack(targets[0], targets[2])); dataNodes[0].updateHeartbeat( 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0); } - private static DatanodeDescriptor[] chooseTarget(int numOfReplicas) { + private static DatanodeStorageInfo[] chooseTarget(int numOfReplicas) { return chooseTarget(numOfReplicas, dataNodes[0]); } - private static DatanodeDescriptor[] chooseTarget(int numOfReplicas, + private static DatanodeStorageInfo[] chooseTarget(int numOfReplicas, DatanodeDescriptor writer) { return chooseTarget(numOfReplicas, writer, - new ArrayList()); + new ArrayList()); } - private static DatanodeDescriptor[] chooseTarget(int numOfReplicas, - List chosenNodes) { + private static DatanodeStorageInfo[] chooseTarget(int numOfReplicas, + List chosenNodes) { return chooseTarget(numOfReplicas, dataNodes[0], chosenNodes); } - private static DatanodeDescriptor[] chooseTarget(int numOfReplicas, - DatanodeDescriptor writer, List chosenNodes) { + private static DatanodeStorageInfo[] chooseTarget(int numOfReplicas, + DatanodeDescriptor writer, List chosenNodes) { return chooseTarget(numOfReplicas, writer, chosenNodes, null); } - private static DatanodeDescriptor[] chooseTarget(int numOfReplicas, - List chosenNodes, Set excludedNodes) { + private static DatanodeStorageInfo[] chooseTarget(int numOfReplicas, + List chosenNodes, Set excludedNodes) { return chooseTarget(numOfReplicas, dataNodes[0], chosenNodes, excludedNodes); } - private static DatanodeDescriptor[] chooseTarget( + private static DatanodeStorageInfo[] chooseTarget( int numOfReplicas, DatanodeDescriptor writer, - List chosenNodes, + List chosenNodes, Set excludedNodes) { return replicator.chooseTarget(filename, numOfReplicas, writer, chosenNodes, - false, excludedNodes, BLOCK_SIZE); + false, excludedNodes, BLOCK_SIZE, StorageType.DEFAULT); } /** @@ -215,8 +230,8 @@ public class TestReplicationPolicy { @Test public void testChooseTarget2() throws Exception { Set excludedNodes; - DatanodeDescriptor[] targets; - List chosenNodes = new ArrayList(); + DatanodeStorageInfo[] targets; + List chosenNodes = new ArrayList(); excludedNodes = new HashSet(); excludedNodes.add(dataNodes[1]); @@ -228,49 +243,52 @@ public class TestReplicationPolicy { excludedNodes.add(dataNodes[1]); targets = chooseTarget(1, chosenNodes, excludedNodes); assertEquals(targets.length, 1); - assertEquals(targets[0], dataNodes[0]); + assertEquals(storages[0], targets[0]); excludedNodes.clear(); chosenNodes.clear(); excludedNodes.add(dataNodes[1]); targets = chooseTarget(2, chosenNodes, excludedNodes); assertEquals(targets.length, 2); - assertEquals(targets[0], dataNodes[0]); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertEquals(storages[0], targets[0]); + + assertFalse(isOnSameRack(targets[0], targets[1])); excludedNodes.clear(); chosenNodes.clear(); excludedNodes.add(dataNodes[1]); targets = chooseTarget(3, chosenNodes, excludedNodes); assertEquals(targets.length, 3); - assertEquals(targets[0], dataNodes[0]); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - assertTrue(cluster.isOnSameRack(targets[1], targets[2])); + assertEquals(storages[0], targets[0]); + + assertFalse(isOnSameRack(targets[0], targets[1])); + assertTrue(isOnSameRack(targets[1], targets[2])); excludedNodes.clear(); chosenNodes.clear(); excludedNodes.add(dataNodes[1]); targets = chooseTarget(4, chosenNodes, excludedNodes); assertEquals(targets.length, 4); - assertEquals(targets[0], dataNodes[0]); + assertEquals(storages[0], targets[0]); + for(int i=1; i<4; i++) { - assertFalse(cluster.isOnSameRack(targets[0], targets[i])); + assertFalse(isOnSameRack(targets[0], targets[i])); } - assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || - cluster.isOnSameRack(targets[2], targets[3])); - assertFalse(cluster.isOnSameRack(targets[1], targets[3])); + assertTrue(isOnSameRack(targets[1], targets[2]) || + isOnSameRack(targets[2], targets[3])); + assertFalse(isOnSameRack(targets[1], targets[3])); excludedNodes.clear(); chosenNodes.clear(); excludedNodes.add(dataNodes[1]); - chosenNodes.add(dataNodes[2]); + chosenNodes.add(storages[2]); targets = replicator.chooseTarget(filename, 1, dataNodes[0], chosenNodes, true, - excludedNodes, BLOCK_SIZE); + excludedNodes, BLOCK_SIZE, StorageType.DEFAULT); System.out.println("targets=" + Arrays.asList(targets)); assertEquals(2, targets.length); //make sure that the chosen node is in the target. int i = 0; - for (; i < targets.length && !dataNodes[2].equals(targets[i]); i++); + for (; i < targets.length && !storages[2].equals(targets[i]); i++); assertTrue(i < targets.length); } @@ -289,34 +307,34 @@ public class TestReplicationPolicy { 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); // no space - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0); assertEquals(targets.length, 0); targets = chooseTarget(1); assertEquals(targets.length, 1); - assertEquals(targets[0], dataNodes[1]); + assertEquals(storages[1], targets[0]); targets = chooseTarget(2); assertEquals(targets.length, 2); - assertEquals(targets[0], dataNodes[1]); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertEquals(storages[1], targets[0]); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3); assertEquals(targets.length, 3); - assertEquals(targets[0], dataNodes[1]); - assertTrue(cluster.isOnSameRack(targets[1], targets[2])); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertEquals(storages[1], targets[0]); + assertTrue(isOnSameRack(targets[1], targets[2])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(4); assertEquals(targets.length, 4); - assertEquals(targets[0], dataNodes[1]); + assertEquals(storages[1], targets[0]); for(int i=1; i<4; i++) { - assertFalse(cluster.isOnSameRack(targets[0], targets[i])); + assertFalse(isOnSameRack(targets[0], targets[i])); } - assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || - cluster.isOnSameRack(targets[2], targets[3])); - assertFalse(cluster.isOnSameRack(targets[1], targets[3])); + assertTrue(isOnSameRack(targets[1], targets[2]) || + isOnSameRack(targets[2], targets[3])); + assertFalse(isOnSameRack(targets[1], targets[3])); dataNodes[0].updateHeartbeat( 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, @@ -340,27 +358,27 @@ public class TestReplicationPolicy { (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); } - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0); assertEquals(targets.length, 0); targets = chooseTarget(1); assertEquals(targets.length, 1); - assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); + assertFalse(isOnSameRack(targets[0], dataNodes[0])); targets = chooseTarget(2); assertEquals(targets.length, 2); - assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertFalse(isOnSameRack(targets[0], dataNodes[0])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3); assertEquals(targets.length, 3); for(int i=0; i<3; i++) { - assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0])); + assertFalse(isOnSameRack(targets[i], dataNodes[0])); } - assertTrue(cluster.isOnSameRack(targets[0], targets[1]) || - cluster.isOnSameRack(targets[1], targets[2])); - assertFalse(cluster.isOnSameRack(targets[0], targets[2])); + assertTrue(isOnSameRack(targets[0], targets[1]) || + isOnSameRack(targets[1], targets[2])); + assertFalse(isOnSameRack(targets[0], targets[2])); for(int i=0; i<2; i++) { dataNodes[i].updateHeartbeat( @@ -381,7 +399,7 @@ public class TestReplicationPolicy { DatanodeDescriptor writerDesc = DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r4"); - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0, writerDesc); assertEquals(targets.length, 0); @@ -390,12 +408,12 @@ public class TestReplicationPolicy { targets = chooseTarget(2, writerDesc); assertEquals(targets.length, 2); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3, writerDesc); assertEquals(targets.length, 3); - assertTrue(cluster.isOnSameRack(targets[1], targets[2])); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertTrue(isOnSameRack(targets[1], targets[2])); + assertFalse(isOnSameRack(targets[0], targets[1])); } /** @@ -436,7 +454,7 @@ public class TestReplicationPolicy { // try to choose NUM_OF_DATANODES which is more than actually available // nodes. - DatanodeDescriptor[] targets = chooseTarget(NUM_OF_DATANODES); + DatanodeStorageInfo[] targets = chooseTarget(NUM_OF_DATANODES); assertEquals(targets.length, NUM_OF_DATANODES - 2); final List log = appender.getLog(); @@ -456,18 +474,30 @@ public class TestReplicationPolicy { } } - private boolean containsWithinRange(DatanodeDescriptor target, + private boolean containsWithinRange(DatanodeStorageInfo target, DatanodeDescriptor[] nodes, int startIndex, int endIndex) { assert startIndex >= 0 && startIndex < nodes.length; assert endIndex >= startIndex && endIndex < nodes.length; for (int i = startIndex; i <= endIndex; i++) { - if (nodes[i].equals(target)) { + if (nodes[i].equals(target.getDatanodeDescriptor())) { return true; } } return false; } + private boolean containsWithinRange(DatanodeDescriptor target, + DatanodeStorageInfo[] nodes, int startIndex, int endIndex) { + assert startIndex >= 0 && startIndex < nodes.length; + assert endIndex >= startIndex && endIndex < nodes.length; + for (int i = startIndex; i <= endIndex; i++) { + if (nodes[i].getDatanodeDescriptor().equals(target)) { + return true; + } + } + return false; + } + @Test public void testChooseTargetWithStaleNodes() throws Exception { // Set dataNodes[0] as stale @@ -476,19 +506,19 @@ public class TestReplicationPolicy { .getDatanodeManager().getHeartbeatManager().heartbeatCheck(); assertTrue(namenode.getNamesystem().getBlockManager() .getDatanodeManager().shouldAvoidStaleDataNodesForWrite()); - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; // We set the datanode[0] as stale, thus should choose datanode[1] since // datanode[1] is on the same rack with datanode[0] (writer) targets = chooseTarget(1); assertEquals(targets.length, 1); - assertEquals(targets[0], dataNodes[1]); + assertEquals(storages[1], targets[0]); Set excludedNodes = new HashSet(); excludedNodes.add(dataNodes[1]); - List chosenNodes = new ArrayList(); + List chosenNodes = new ArrayList(); targets = chooseTarget(1, chosenNodes, excludedNodes); assertEquals(targets.length, 1); - assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); + assertFalse(isOnSameRack(targets[0], dataNodes[0])); // reset dataNodes[0].setLastUpdate(Time.now()); @@ -513,7 +543,7 @@ public class TestReplicationPolicy { namenode.getNamesystem().getBlockManager() .getDatanodeManager().getHeartbeatManager().heartbeatCheck(); - DatanodeDescriptor[] targets = chooseTarget(0); + DatanodeStorageInfo[] targets = chooseTarget(0); assertEquals(targets.length, 0); // Since we have 6 datanodes total, stale nodes should @@ -585,11 +615,12 @@ public class TestReplicationPolicy { .getDatanode(miniCluster.getDataNodes().get(0).getDatanodeId()); BlockPlacementPolicy replicator = miniCluster.getNameNode() .getNamesystem().getBlockManager().getBlockPlacementPolicy(); - DatanodeDescriptor[] targets = replicator.chooseTarget(filename, 3, - staleNodeInfo, new ArrayList(), false, null, BLOCK_SIZE); + DatanodeStorageInfo[] targets = replicator.chooseTarget(filename, 3, + staleNodeInfo, new ArrayList(), false, null, + BLOCK_SIZE, StorageType.DEFAULT); assertEquals(targets.length, 3); - assertFalse(cluster.isOnSameRack(targets[0], staleNodeInfo)); + assertFalse(isOnSameRack(targets[0], staleNodeInfo)); // Step 2. Set more than half of the datanodes as stale for (int i = 0; i < 4; i++) { @@ -610,10 +641,11 @@ public class TestReplicationPolicy { assertFalse(miniCluster.getNameNode().getNamesystem().getBlockManager() .getDatanodeManager().shouldAvoidStaleDataNodesForWrite()); // Call chooseTarget - targets = replicator.chooseTarget(filename, 3, - staleNodeInfo, new ArrayList(), false, null, BLOCK_SIZE); + targets = replicator.chooseTarget(filename, 3, staleNodeInfo, + new ArrayList(), false, null, BLOCK_SIZE, + StorageType.DEFAULT); assertEquals(targets.length, 3); - assertTrue(cluster.isOnSameRack(targets[0], staleNodeInfo)); + assertTrue(isOnSameRack(targets[0], staleNodeInfo)); // Step 3. Set 2 stale datanodes back to healthy nodes, // still have 2 stale nodes @@ -635,7 +667,7 @@ public class TestReplicationPolicy { // Call chooseTarget targets = chooseTarget(3, staleNodeInfo); assertEquals(targets.length, 3); - assertFalse(cluster.isOnSameRack(targets[0], staleNodeInfo)); + assertFalse(isOnSameRack(targets[0], staleNodeInfo)); } finally { miniCluster.shutdown(); } @@ -650,26 +682,26 @@ public class TestReplicationPolicy { */ @Test public void testRereplicate1() throws Exception { - List chosenNodes = new ArrayList(); - chosenNodes.add(dataNodes[0]); - DatanodeDescriptor[] targets; + List chosenNodes = new ArrayList(); + chosenNodes.add(storages[0]); + DatanodeStorageInfo[] targets; targets = chooseTarget(0, chosenNodes); assertEquals(targets.length, 0); targets = chooseTarget(1, chosenNodes); assertEquals(targets.length, 1); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(targets[0], dataNodes[0])); targets = chooseTarget(2, chosenNodes); assertEquals(targets.length, 2); - assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertTrue(isOnSameRack(targets[0], dataNodes[0])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3, chosenNodes); assertEquals(targets.length, 3); - assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameRack(targets[0], targets[2])); + assertTrue(isOnSameRack(targets[0], dataNodes[0])); + assertFalse(isOnSameRack(targets[0], targets[2])); } /** @@ -681,22 +713,22 @@ public class TestReplicationPolicy { */ @Test public void testRereplicate2() throws Exception { - List chosenNodes = new ArrayList(); - chosenNodes.add(dataNodes[0]); - chosenNodes.add(dataNodes[1]); + List chosenNodes = new ArrayList(); + chosenNodes.add(storages[0]); + chosenNodes.add(storages[1]); - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0, chosenNodes); assertEquals(targets.length, 0); targets = chooseTarget(1, chosenNodes); assertEquals(targets.length, 1); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(targets[0], dataNodes[0])); targets = chooseTarget(2, chosenNodes); assertEquals(targets.length, 2); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[1])); + assertFalse(isOnSameRack(targets[0], dataNodes[0])); + assertFalse(isOnSameRack(targets[1], dataNodes[0])); } /** @@ -708,31 +740,31 @@ public class TestReplicationPolicy { */ @Test public void testRereplicate3() throws Exception { - List chosenNodes = new ArrayList(); - chosenNodes.add(dataNodes[0]); - chosenNodes.add(dataNodes[2]); + List chosenNodes = new ArrayList(); + chosenNodes.add(storages[0]); + chosenNodes.add(storages[2]); - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0, chosenNodes); assertEquals(targets.length, 0); targets = chooseTarget(1, chosenNodes); assertEquals(targets.length, 1); - assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameRack(dataNodes[2], targets[0])); + assertTrue(isOnSameRack(targets[0], dataNodes[0])); + assertFalse(isOnSameRack(targets[0], dataNodes[2])); targets = chooseTarget(1, dataNodes[2], chosenNodes); assertEquals(targets.length, 1); - assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0])); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); + assertTrue(isOnSameRack(targets[0], dataNodes[2])); + assertFalse(isOnSameRack(targets[0], dataNodes[0])); targets = chooseTarget(2, chosenNodes); assertEquals(targets.length, 2); - assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); + assertTrue(isOnSameRack(targets[0], dataNodes[0])); targets = chooseTarget(2, dataNodes[2], chosenNodes); assertEquals(targets.length, 2); - assertTrue(cluster.isOnSameRack(dataNodes[2], targets[0])); + assertTrue(isOnSameRack(targets[0], dataNodes[2])); } /** @@ -1174,4 +1206,4 @@ public class TestReplicationPolicy { chosenBlocks = underReplicatedBlocks.chooseUnderReplicatedBlocks(1); assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0); } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java index 5670c2074f9..f38b4de21d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.net.NetworkTopology; @@ -57,41 +58,57 @@ public class TestReplicationPolicyWithNodeGroup { private BlockPlacementPolicy replicator; private static final String filename = "/dummyfile.txt"; - private final static DatanodeDescriptor dataNodes[] = new DatanodeDescriptor[] { - DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1/r1/n2"), - DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/d1/r2/n3"), - DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/d1/r2/n3"), - DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d1/r2/n4"), - DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r3/n5"), - DFSTestUtil.getDatanodeDescriptor("8.8.8.8", "/d2/r3/n6") - }; - - private final static DatanodeDescriptor dataNodesInBoundaryCase[] = - new DatanodeDescriptor[] { - DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/d1/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/d1/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/d1/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/d1/r1/n2"), - DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/d1/r2/n3"), - DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d1/r2/n3") - }; - - private final static DatanodeDescriptor dataNodesInMoreTargetsCase[] = - new DatanodeDescriptor[] { - DFSTestUtil.getDatanodeDescriptor("1.1.1.1", "/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("2.2.2.2", "/r1/n1"), - DFSTestUtil.getDatanodeDescriptor("3.3.3.3", "/r1/n2"), - DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/r1/n2"), - DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/r1/n3"), - DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/r1/n3"), - DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/r2/n4"), - DFSTestUtil.getDatanodeDescriptor("8.8.8.8", "/r2/n4"), - DFSTestUtil.getDatanodeDescriptor("9.9.9.9", "/r2/n5"), - DFSTestUtil.getDatanodeDescriptor("10.10.10.10", "/r2/n5"), - DFSTestUtil.getDatanodeDescriptor("11.11.11.11", "/r2/n6"), - DFSTestUtil.getDatanodeDescriptor("12.12.12.12", "/r2/n6"), + private static final DatanodeStorageInfo[] storages; + private static final DatanodeDescriptor[] dataNodes; + static { + final String[] racks = { + "/d1/r1/n1", + "/d1/r1/n1", + "/d1/r1/n2", + "/d1/r2/n3", + "/d1/r2/n3", + "/d1/r2/n4", + "/d2/r3/n5", + "/d2/r3/n6" + }; + storages = DFSTestUtil.createDatanodeStorageInfos(racks); + dataNodes = DFSTestUtil.toDatanodeDescriptor(storages); + } + + private static final DatanodeStorageInfo[] storagesInBoundaryCase; + private static final DatanodeDescriptor[] dataNodesInBoundaryCase; + static { + final String[] racksInBoundaryCase = { + "/d1/r1/n1", + "/d1/r1/n1", + "/d1/r1/n1", + "/d1/r1/n2", + "/d1/r2/n3", + "/d1/r2/n3" + }; + storagesInBoundaryCase = DFSTestUtil.createDatanodeStorageInfos(racksInBoundaryCase); + dataNodesInBoundaryCase = DFSTestUtil.toDatanodeDescriptor(storagesInBoundaryCase); + } + + private static final DatanodeStorageInfo[] storagesInMoreTargetsCase; + private final static DatanodeDescriptor[] dataNodesInMoreTargetsCase; + static { + final String[] racksInMoreTargetsCase = { + "/r1/n1", + "/r1/n1", + "/r1/n2", + "/r1/n2", + "/r1/n3", + "/r1/n3", + "/r2/n4", + "/r2/n4", + "/r2/n5", + "/r2/n5", + "/r2/n6", + "/r2/n6" + }; + storagesInMoreTargetsCase = DFSTestUtil.createDatanodeStorageInfos(racksInMoreTargetsCase); + dataNodesInMoreTargetsCase = DFSTestUtil.toDatanodeDescriptor(storagesInMoreTargetsCase); }; private final static DatanodeDescriptor NODE = @@ -142,11 +159,12 @@ public class TestReplicationPolicyWithNodeGroup { * Return false if two targets are found on the same NodeGroup. */ private static boolean checkTargetsOnDifferentNodeGroup( - DatanodeDescriptor[] targets) { + DatanodeStorageInfo[] targets) { if(targets.length == 0) return true; Set targetSet = new HashSet(); - for(DatanodeDescriptor node:targets) { + for(DatanodeStorageInfo storage:targets) { + final DatanodeDescriptor node = storage.getDatanodeDescriptor(); String nodeGroup = NetworkTopology.getLastHalf(node.getNetworkLocation()); if(targetSet.contains(nodeGroup)) { return false; @@ -156,34 +174,50 @@ public class TestReplicationPolicyWithNodeGroup { } return true; } - - private DatanodeDescriptor[] chooseTarget(int numOfReplicas) { + + private boolean isOnSameRack(DatanodeStorageInfo left, DatanodeStorageInfo right) { + return isOnSameRack(left.getDatanodeDescriptor(), right); + } + + private boolean isOnSameRack(DatanodeDescriptor left, DatanodeStorageInfo right) { + return cluster.isOnSameRack(left, right.getDatanodeDescriptor()); + } + + private boolean isOnSameNodeGroup(DatanodeStorageInfo left, DatanodeStorageInfo right) { + return isOnSameNodeGroup(left.getDatanodeDescriptor(), right); + } + + private boolean isOnSameNodeGroup(DatanodeDescriptor left, DatanodeStorageInfo right) { + return cluster.isOnSameNodeGroup(left, right.getDatanodeDescriptor()); + } + + private DatanodeStorageInfo[] chooseTarget(int numOfReplicas) { return chooseTarget(numOfReplicas, dataNodes[0]); } - private DatanodeDescriptor[] chooseTarget(int numOfReplicas, + private DatanodeStorageInfo[] chooseTarget(int numOfReplicas, DatanodeDescriptor writer) { return chooseTarget(numOfReplicas, writer, - new ArrayList()); + new ArrayList()); } - private DatanodeDescriptor[] chooseTarget(int numOfReplicas, - List chosenNodes) { + private DatanodeStorageInfo[] chooseTarget(int numOfReplicas, + List chosenNodes) { return chooseTarget(numOfReplicas, dataNodes[0], chosenNodes); } - private DatanodeDescriptor[] chooseTarget(int numOfReplicas, - DatanodeDescriptor writer, List chosenNodes) { + private DatanodeStorageInfo[] chooseTarget(int numOfReplicas, + DatanodeDescriptor writer, List chosenNodes) { return chooseTarget(numOfReplicas, writer, chosenNodes, null); } - private DatanodeDescriptor[] chooseTarget( + private DatanodeStorageInfo[] chooseTarget( int numOfReplicas, DatanodeDescriptor writer, - List chosenNodes, + List chosenNodes, Set excludedNodes) { return replicator.chooseTarget(filename, numOfReplicas, writer, chosenNodes, - false, excludedNodes, BLOCK_SIZE); + false, excludedNodes, BLOCK_SIZE, StorageType.DEFAULT); } /** @@ -201,32 +235,36 @@ public class TestReplicationPolicyWithNodeGroup { 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 4, 0); // overloaded - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0); assertEquals(targets.length, 0); targets = chooseTarget(1); assertEquals(targets.length, 1); - assertEquals(targets[0], dataNodes[0]); + assertEquals(storages[0], targets[0]); + targets = chooseTarget(2); assertEquals(targets.length, 2); - assertEquals(targets[0], dataNodes[0]); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertEquals(storages[0], targets[0]); + + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3); assertEquals(targets.length, 3); - assertEquals(targets[0], dataNodes[0]); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - assertTrue(cluster.isOnSameRack(targets[1], targets[2])); - assertFalse(cluster.isOnSameNodeGroup(targets[1], targets[2])); + assertEquals(storages[0], targets[0]); + + assertFalse(isOnSameRack(targets[0], targets[1])); + assertTrue(isOnSameRack(targets[1], targets[2])); + assertFalse(isOnSameNodeGroup(targets[1], targets[2])); targets = chooseTarget(4); assertEquals(targets.length, 4); - assertEquals(targets[0], dataNodes[0]); - assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || - cluster.isOnSameRack(targets[2], targets[3])); - assertFalse(cluster.isOnSameRack(targets[0], targets[2])); + assertEquals(storages[0], targets[0]); + + assertTrue(isOnSameRack(targets[1], targets[2]) || + isOnSameRack(targets[2], targets[3])); + assertFalse(isOnSameRack(targets[0], targets[2])); // Make sure no more than one replicas are on the same nodegroup verifyNoTwoTargetsOnSameNodeGroup(targets); @@ -235,10 +273,10 @@ public class TestReplicationPolicyWithNodeGroup { HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0); } - private void verifyNoTwoTargetsOnSameNodeGroup(DatanodeDescriptor[] targets) { + private void verifyNoTwoTargetsOnSameNodeGroup(DatanodeStorageInfo[] targets) { Set nodeGroupSet = new HashSet(); - for (DatanodeDescriptor target: targets) { - nodeGroupSet.add(target.getNetworkLocation()); + for (DatanodeStorageInfo target: targets) { + nodeGroupSet.add(target.getDatanodeDescriptor().getNetworkLocation()); } assertEquals(nodeGroupSet.size(), targets.length); } @@ -253,36 +291,37 @@ public class TestReplicationPolicyWithNodeGroup { */ @Test public void testChooseTarget2() throws Exception { - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; BlockPlacementPolicyDefault repl = (BlockPlacementPolicyDefault)replicator; - List chosenNodes = new ArrayList(); + List chosenNodes = new ArrayList(); Set excludedNodes = new HashSet(); excludedNodes.add(dataNodes[1]); targets = repl.chooseTarget(filename, 4, dataNodes[0], chosenNodes, false, - excludedNodes, BLOCK_SIZE); + excludedNodes, BLOCK_SIZE, StorageType.DEFAULT); assertEquals(targets.length, 4); - assertEquals(targets[0], dataNodes[0]); + assertEquals(storages[0], targets[0]); + assertTrue(cluster.isNodeGroupAware()); // Make sure no replicas are on the same nodegroup for (int i=1;i<4;i++) { - assertFalse(cluster.isOnSameNodeGroup(targets[0], targets[i])); + assertFalse(isOnSameNodeGroup(targets[0], targets[i])); } - assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || - cluster.isOnSameRack(targets[2], targets[3])); - assertFalse(cluster.isOnSameRack(targets[1], targets[3])); + assertTrue(isOnSameRack(targets[1], targets[2]) || + isOnSameRack(targets[2], targets[3])); + assertFalse(isOnSameRack(targets[1], targets[3])); excludedNodes.clear(); chosenNodes.clear(); excludedNodes.add(dataNodes[1]); - chosenNodes.add(dataNodes[2]); + chosenNodes.add(storages[2]); targets = repl.chooseTarget(filename, 1, dataNodes[0], chosenNodes, true, - excludedNodes, BLOCK_SIZE); + excludedNodes, BLOCK_SIZE, StorageType.DEFAULT); System.out.println("targets=" + Arrays.asList(targets)); assertEquals(2, targets.length); //make sure that the chosen node is in the target. int i = 0; - for(; i < targets.length && !dataNodes[2].equals(targets[i]); i++); + for(; i < targets.length && !storages[2].equals(targets[i]); i++); assertTrue(i < targets.length); } @@ -301,32 +340,32 @@ public class TestReplicationPolicyWithNodeGroup { 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); // no space - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0); assertEquals(targets.length, 0); targets = chooseTarget(1); assertEquals(targets.length, 1); - assertEquals(targets[0], dataNodes[1]); + assertEquals(storages[1], targets[0]); targets = chooseTarget(2); assertEquals(targets.length, 2); - assertEquals(targets[0], dataNodes[1]); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertEquals(storages[1], targets[0]); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3); assertEquals(targets.length, 3); - assertEquals(targets[0], dataNodes[1]); - assertTrue(cluster.isOnSameRack(targets[1], targets[2])); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertEquals(storages[1], targets[0]); + assertTrue(isOnSameRack(targets[1], targets[2])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(4); assertEquals(targets.length, 4); - assertEquals(targets[0], dataNodes[1]); + assertEquals(storages[1], targets[0]); assertTrue(cluster.isNodeGroupAware()); verifyNoTwoTargetsOnSameNodeGroup(targets); - assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || - cluster.isOnSameRack(targets[2], targets[3])); + assertTrue(isOnSameRack(targets[1], targets[2]) || + isOnSameRack(targets[2], targets[3])); dataNodes[0].updateHeartbeat( 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, @@ -351,28 +390,28 @@ public class TestReplicationPolicyWithNodeGroup { (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); } - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0); assertEquals(targets.length, 0); targets = chooseTarget(1); assertEquals(targets.length, 1); - assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); + assertFalse(isOnSameRack(dataNodes[0], targets[0])); targets = chooseTarget(2); assertEquals(targets.length, 2); - assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertFalse(isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3); assertEquals(targets.length, 3); for(int i=0; i<3; i++) { - assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0])); + assertFalse(isOnSameRack(dataNodes[0], targets[i])); } verifyNoTwoTargetsOnSameNodeGroup(targets); - assertTrue(cluster.isOnSameRack(targets[0], targets[1]) || - cluster.isOnSameRack(targets[1], targets[2])); - assertFalse(cluster.isOnSameRack(targets[0], targets[2])); + assertTrue(isOnSameRack(targets[0], targets[1]) || + isOnSameRack(targets[1], targets[2])); + assertFalse(isOnSameRack(targets[0], targets[2])); } /** @@ -385,7 +424,7 @@ public class TestReplicationPolicyWithNodeGroup { @Test public void testChooseTarget5() throws Exception { setupDataNodeCapacity(); - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0, NODE); assertEquals(targets.length, 0); @@ -394,12 +433,12 @@ public class TestReplicationPolicyWithNodeGroup { targets = chooseTarget(2, NODE); assertEquals(targets.length, 2); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3, NODE); assertEquals(targets.length, 3); - assertTrue(cluster.isOnSameRack(targets[1], targets[2])); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertTrue(isOnSameRack(targets[1], targets[2])); + assertFalse(isOnSameRack(targets[0], targets[1])); verifyNoTwoTargetsOnSameNodeGroup(targets); } @@ -413,27 +452,27 @@ public class TestReplicationPolicyWithNodeGroup { @Test public void testRereplicate1() throws Exception { setupDataNodeCapacity(); - List chosenNodes = new ArrayList(); - chosenNodes.add(dataNodes[0]); - DatanodeDescriptor[] targets; + List chosenNodes = new ArrayList(); + chosenNodes.add(storages[0]); + DatanodeStorageInfo[] targets; targets = chooseTarget(0, chosenNodes); assertEquals(targets.length, 0); targets = chooseTarget(1, chosenNodes); assertEquals(targets.length, 1); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(dataNodes[0], targets[0])); targets = chooseTarget(2, chosenNodes); assertEquals(targets.length, 2); - assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertTrue(isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3, chosenNodes); assertEquals(targets.length, 3); - assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameNodeGroup(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameRack(targets[0], targets[2])); + assertTrue(isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameNodeGroup(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(targets[0], targets[2])); } /** @@ -446,22 +485,22 @@ public class TestReplicationPolicyWithNodeGroup { @Test public void testRereplicate2() throws Exception { setupDataNodeCapacity(); - List chosenNodes = new ArrayList(); - chosenNodes.add(dataNodes[0]); - chosenNodes.add(dataNodes[1]); + List chosenNodes = new ArrayList(); + chosenNodes.add(storages[0]); + chosenNodes.add(storages[1]); - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0, chosenNodes); assertEquals(targets.length, 0); targets = chooseTarget(1, chosenNodes); assertEquals(targets.length, 1); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(dataNodes[0], targets[0])); targets = chooseTarget(2, chosenNodes); assertEquals(targets.length, 2); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]) && - cluster.isOnSameRack(dataNodes[0], targets[1])); + assertFalse(isOnSameRack(dataNodes[0], targets[0]) && + isOnSameRack(dataNodes[0], targets[1])); } /** @@ -474,33 +513,33 @@ public class TestReplicationPolicyWithNodeGroup { @Test public void testRereplicate3() throws Exception { setupDataNodeCapacity(); - List chosenNodes = new ArrayList(); - chosenNodes.add(dataNodes[0]); - chosenNodes.add(dataNodes[3]); + List chosenNodes = new ArrayList(); + chosenNodes.add(storages[0]); + chosenNodes.add(storages[3]); - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0, chosenNodes); assertEquals(targets.length, 0); targets = chooseTarget(1, chosenNodes); assertEquals(targets.length, 1); - assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameRack(dataNodes[3], targets[0])); + assertTrue(isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameRack(dataNodes[3], targets[0])); targets = chooseTarget(1, dataNodes[3], chosenNodes); assertEquals(targets.length, 1); - assertTrue(cluster.isOnSameRack(dataNodes[3], targets[0])); - assertFalse(cluster.isOnSameNodeGroup(dataNodes[3], targets[0])); - assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); + assertTrue(isOnSameRack(dataNodes[3], targets[0])); + assertFalse(isOnSameNodeGroup(dataNodes[3], targets[0])); + assertFalse(isOnSameRack(dataNodes[0], targets[0])); targets = chooseTarget(2, chosenNodes); assertEquals(targets.length, 2); - assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); - assertFalse(cluster.isOnSameNodeGroup(dataNodes[0], targets[0])); + assertTrue(isOnSameRack(dataNodes[0], targets[0])); + assertFalse(isOnSameNodeGroup(dataNodes[0], targets[0])); targets = chooseTarget(2, dataNodes[3], chosenNodes); assertEquals(targets.length, 2); - assertTrue(cluster.isOnSameRack(dataNodes[3], targets[0])); + assertTrue(isOnSameRack(dataNodes[3], targets[0])); } /** @@ -583,7 +622,7 @@ public class TestReplicationPolicyWithNodeGroup { 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0); } - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; targets = chooseTarget(0, dataNodesInBoundaryCase[0]); assertEquals(targets.length, 0); @@ -592,7 +631,7 @@ public class TestReplicationPolicyWithNodeGroup { targets = chooseTarget(2, dataNodesInBoundaryCase[0]); assertEquals(targets.length, 2); - assertFalse(cluster.isOnSameRack(targets[0], targets[1])); + assertFalse(isOnSameRack(targets[0], targets[1])); targets = chooseTarget(3, dataNodesInBoundaryCase[0]); assertEquals(targets.length, 3); @@ -613,15 +652,13 @@ public class TestReplicationPolicyWithNodeGroup { 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0); } - List chosenNodes = new ArrayList(); - chosenNodes.add(dataNodesInBoundaryCase[0]); - chosenNodes.add(dataNodesInBoundaryCase[5]); - DatanodeDescriptor[] targets; + List chosenNodes = new ArrayList(); + chosenNodes.add(storagesInBoundaryCase[0]); + chosenNodes.add(storagesInBoundaryCase[5]); + DatanodeStorageInfo[] targets; targets = chooseTarget(1, dataNodesInBoundaryCase[0], chosenNodes); - assertFalse(cluster.isOnSameNodeGroup(targets[0], - dataNodesInBoundaryCase[0])); - assertFalse(cluster.isOnSameNodeGroup(targets[0], - dataNodesInBoundaryCase[5])); + assertFalse(isOnSameNodeGroup(dataNodesInBoundaryCase[0], targets[0])); + assertFalse(isOnSameNodeGroup(dataNodesInBoundaryCase[5], targets[0])); assertTrue(checkTargetsOnDifferentNodeGroup(targets)); } @@ -654,7 +691,7 @@ public class TestReplicationPolicyWithNodeGroup { 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0); } - DatanodeDescriptor[] targets; + DatanodeStorageInfo[] targets; // Test normal case -- 3 replicas targets = chooseTarget(3, dataNodesInMoreTargetsCase[0]); assertEquals(targets.length, 3); From eb9f1b670726e1af03f2e940ce2696b880964972 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 20 Sep 2013 22:06:09 +0000 Subject: [PATCH 13/76] HDFS-5232. Protocol changes to transmit StorageUuid. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1525153 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 ++ ...amenodeProtocolServerSideTranslatorPB.java | 2 +- .../ClientNamenodeProtocolTranslatorPB.java | 2 +- ...atanodeProtocolClientSideTranslatorPB.java | 2 +- ...atanodeProtocolServerSideTranslatorPB.java | 4 +-- ...atanodeProtocolServerSideTranslatorPB.java | 2 +- .../InterDatanodeProtocolTranslatorPB.java | 2 +- .../hadoop/hdfs/protocolPB/PBHelper.java | 36 +++++++++---------- .../main/proto/ClientNamenodeProtocol.proto | 2 +- .../src/main/proto/DatanodeProtocol.proto | 8 ++--- .../main/proto/InterDatanodeProtocol.proto | 2 +- .../hadoop-hdfs/src/main/proto/hdfs.proto | 11 +++--- 12 files changed, 40 insertions(+), 35 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index b9d4981a816..9f4e314ad75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -26,3 +26,5 @@ IMPROVEMENTS: HDFS-4990. Change BlockPlacementPolicy to choose storages instead of datanodes. (szetszwo) + + HDFS-5232. Protocol changes to transmit StorageUuid. (Arpit Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 6e8e7bc34eb..8290a4b4e43 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -405,7 +405,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements throws ServiceException { try { List existingList = req.getExistingsList(); - List existingStorageIDsList = req.getExistingStorageIDsList(); + List existingStorageIDsList = req.getExistingStorageUuidsList(); List excludesList = req.getExcludesList(); LocatedBlock result = server.getAdditionalDatanode(req.getSrc(), PBHelper.convert(req.getBlk()), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 84d10e473a6..30ea3e562e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -345,7 +345,7 @@ public class ClientNamenodeProtocolTranslatorPB implements .setSrc(src) .setBlk(PBHelper.convert(blk)) .addAllExistings(PBHelper.convert(existings)) - .addAllExistingStorageIDs(Arrays.asList(existingStorageIDs)) + .addAllExistingStorageUuids(Arrays.asList(existingStorageIDs)) .addAllExcludes(PBHelper.convert(excludes)) .setNumAdditionalNodes(numAdditionalNodes) .setClientName(clientName) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java index fd4cc4b01c5..315ad92d049 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java @@ -213,7 +213,7 @@ public class DatanodeProtocolClientSideTranslatorPB implements for (StorageReceivedDeletedBlocks storageBlock : receivedAndDeletedBlocks) { StorageReceivedDeletedBlocksProto.Builder repBuilder = StorageReceivedDeletedBlocksProto.newBuilder(); - repBuilder.setStorageID(storageBlock.getStorageID()); + repBuilder.setStorageUuid(storageBlock.getStorageID()); for (ReceivedDeletedBlockInfo rdBlock : storageBlock.getBlocks()) { repBuilder.addBlocks(PBHelper.convert(rdBlock)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java index 3e424602fa9..9a63d37dec6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java @@ -104,7 +104,7 @@ public class DatanodeProtocolServerSideTranslatorPB implements StorageReport[] report = new StorageReport[list.size()]; int i = 0; for (StorageReportProto p : list) { - report[i++] = new StorageReport(p.getStorageID(), p.getFailed(), + report[i++] = new StorageReport(p.getStorageUuid(), p.getFailed(), p.getCapacity(), p.getDfsUsed(), p.getRemaining(), p.getBlockPoolUsed()); } @@ -174,7 +174,7 @@ public class DatanodeProtocolServerSideTranslatorPB implements for (int j = 0; j < list.size(); j++) { rdBlocks[j] = PBHelper.convert(list.get(j)); } - info[i] = new StorageReceivedDeletedBlocks(sBlock.getStorageID(), rdBlocks); + info[i] = new StorageReceivedDeletedBlocks(sBlock.getStorageUuid(), rdBlocks); } try { impl.blockReceivedAndDeleted(PBHelper.convert(request.getRegistration()), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java index 8f3eed96852..087c697c587 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java @@ -82,6 +82,6 @@ public class InterDatanodeProtocolServerSideTranslatorPB implements throw new ServiceException(e); } return UpdateReplicaUnderRecoveryResponseProto.newBuilder() - .setStorageID(storageID).build(); + .setStorageUuid(storageID).build(); } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java index 4e518c35bfa..5174d861882 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java @@ -109,7 +109,7 @@ public class InterDatanodeProtocolTranslatorPB implements .setNewLength(newLength).setRecoveryId(recoveryId).build(); try { return rpcProxy.updateReplicaUnderRecovery(NULL_CONTROLLER, req - ).getStorageID(); + ).getStorageUuid(); } catch (ServiceException e) { throw ProtobufHelper.getRemoteException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index d118904d0ca..c2de12e7dce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -94,7 +94,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.File import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto.Builder; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlocksProto; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageIDsProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeCommandProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto; @@ -226,7 +226,7 @@ public class PBHelper { // DatanodeId public static DatanodeID convert(DatanodeIDProto dn) { - return new DatanodeID(dn.getIpAddr(), dn.getHostName(), dn.getStorageID(), + return new DatanodeID(dn.getIpAddr(), dn.getHostName(), dn.getDatanodeUuid(), dn.getXferPort(), dn.getInfoPort(), dn.getIpcPort()); } @@ -234,7 +234,7 @@ public class PBHelper { return DatanodeIDProto.newBuilder() .setIpAddr(dn.getIpAddr()) .setHostName(dn.getHostName()) - .setStorageID(dn.getStorageID()) + .setDatanodeUuid(dn.getStorageID()) .setXferPort(dn.getXferPort()) .setInfoPort(dn.getInfoPort()) .setIpcPort(dn.getIpcPort()).build(); @@ -276,11 +276,11 @@ public class PBHelper { public static BlockWithLocationsProto convert(BlockWithLocations blk) { return BlockWithLocationsProto.newBuilder() .setBlock(convert(blk.getBlock())) - .addAllStorageIDs(Arrays.asList(blk.getStorageIDs())).build(); + .addAllStorageUuids(Arrays.asList(blk.getStorageIDs())).build(); } public static BlockWithLocations convert(BlockWithLocationsProto b) { - return new BlockWithLocations(convert(b.getBlock()), b.getStorageIDsList() + return new BlockWithLocations(convert(b.getBlock()), b.getStorageUuidsList() .toArray(new String[0])); } @@ -746,7 +746,7 @@ public class PBHelper { builder.addBlocks(PBHelper.convert(blocks[i])); } builder.addAllTargets(convert(cmd.getTargets())) - .addAllTargetStorageIDs(convert(cmd.getTargetStorageIDs())); + .addAllTargetStorageUuids(convert(cmd.getTargetStorageIDs())); return builder.build(); } @@ -759,11 +759,11 @@ public class PBHelper { return Arrays.asList(ret); } - private static List convert(String[][] targetStorageIDs) { - StorageIDsProto[] ret = new StorageIDsProto[targetStorageIDs.length]; - for (int i = 0; i < targetStorageIDs.length; i++) { - ret[i] = StorageIDsProto.newBuilder() - .addAllStorageIDs(Arrays.asList(targetStorageIDs[i])).build(); + private static List convert(String[][] targetStorageUuids) { + StorageUuidsProto[] ret = new StorageUuidsProto[targetStorageUuids.length]; + for (int i = 0; i < targetStorageUuids.length; i++) { + ret[i] = StorageUuidsProto.newBuilder() + .addAllStorageUuids(Arrays.asList(targetStorageUuids[i])).build(); } return Arrays.asList(ret); } @@ -843,10 +843,10 @@ public class PBHelper { targets[i] = PBHelper.convert(targetList.get(i)); } - List targetStorageIDsList = blkCmd.getTargetStorageIDsList(); - String[][] targetStorageIDs = new String[targetStorageIDsList.size()][]; + List targetStorageUuidsList = blkCmd.getTargetStorageUuidsList(); + String[][] targetStorageIDs = new String[targetStorageUuidsList.size()][]; for(int i = 0; i < targetStorageIDs.length; i++) { - List storageIDs = targetStorageIDsList.get(i).getStorageIDsList(); + List storageIDs = targetStorageUuidsList.get(i).getStorageUuidsList(); targetStorageIDs[i] = storageIDs.toArray(new String[storageIDs.size()]); } @@ -1375,7 +1375,7 @@ public class PBHelper { return DatanodeStorageProto.newBuilder() .setState(PBHelper.convertState(s.getState())) .setStorageType(PBHelper.convertStorageType(s.getStorageType())) - .setStorageID(s.getStorageID()).build(); + .setStorageUuid(s.getStorageID()).build(); } private static StorageState convertState(State state) { @@ -1406,11 +1406,11 @@ public class PBHelper { public static DatanodeStorage convert(DatanodeStorageProto s) { if (s.hasStorageType()) { - return new DatanodeStorage(s.getStorageID(), + return new DatanodeStorage(s.getStorageUuid(), PBHelper.convertState(s.getState()), PBHelper.convertType(s.getStorageType())); } else { - return new DatanodeStorage(s.getStorageID(), + return new DatanodeStorage(s.getStorageUuid(), PBHelper.convertState(s.getState())); } } @@ -1440,7 +1440,7 @@ public class PBHelper { return StorageReportProto.newBuilder() .setBlockPoolUsed(r.getBlockPoolUsed()).setCapacity(r.getCapacity()) .setDfsUsed(r.getDfsUsed()).setRemaining(r.getRemaining()) - .setStorageID(r.getStorageID()).build(); + .setStorageUuid(r.getStorageID()).build(); } public static JournalInfo convert(JournalInfoProto info) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index 968b0740355..fda60857ce1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -141,7 +141,7 @@ message GetAdditionalDatanodeRequestProto { repeated DatanodeInfoProto excludes = 4; required uint32 numAdditionalNodes = 5; required string clientName = 6; - repeated string existingStorageIDs = 7; + repeated string existingStorageUuids = 7; } message GetAdditionalDatanodeResponseProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto index 2f845e08a43..6f93afcc96b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto @@ -53,7 +53,7 @@ message DatanodeStorageProto { READ_ONLY = 1; } - required string storageID = 1; // Unique identifier for the storage + required string storageUuid = 1; optional StorageState state = 2 [default = NORMAL]; optional StorageTypeProto storageType = 3; } @@ -110,7 +110,7 @@ message BlockCommandProto { required string blockPoolId = 2; repeated BlockProto blocks = 3; repeated DatanodeInfosProto targets = 4; - repeated StorageIDsProto targetStorageIDs = 5; + repeated StorageUuidsProto targetStorageUuids = 5; } /** @@ -176,7 +176,7 @@ message HeartbeatRequestProto { } message StorageReportProto { - required string storageID = 1; + required string storageUuid = 1; optional bool failed = 2 [ default = false ]; optional uint64 capacity = 3 [ default = 0 ]; optional uint64 dfsUsed = 4 [ default = 0 ]; @@ -250,7 +250,7 @@ message ReceivedDeletedBlockInfoProto { * List of blocks received and deleted for a storage. */ message StorageReceivedDeletedBlocksProto { - required string storageID = 1; + required string storageUuid = 1; repeated ReceivedDeletedBlockInfoProto blocks = 2; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto index c76f7edfa86..47f79bed169 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto @@ -65,7 +65,7 @@ message UpdateReplicaUnderRecoveryRequestProto { * Response returns updated block information */ message UpdateReplicaUnderRecoveryResponseProto { - required string storageID = 1; // ID of the storage that stores replica + optional string storageUuid = 1; // ID of the storage that stores replica } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 22121a73b5b..b8775259f8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -50,7 +50,10 @@ message ExtendedBlockProto { message DatanodeIDProto { required string ipAddr = 1; // IP address required string hostName = 2; // hostname - required string storageID = 3; // unique storage id + required string datanodeUuid = 3; // UUID assigned to the Datanode. For + // upgraded clusters this is the same + // as the original StorageID of the + // Datanode. required uint32 xferPort = 4; // data streaming port required uint32 infoPort = 5; // info server port required uint32 ipcPort = 6; // ipc server port @@ -124,8 +127,8 @@ enum StorageTypeProto { /** * A list of storage IDs. */ -message StorageIDsProto { - repeated string storageIDs = 1; +message StorageUuidsProto { + repeated string storageUuids = 1; } /** @@ -348,7 +351,7 @@ message BlockProto { */ message BlockWithLocationsProto { required BlockProto block = 1; // Block - repeated string storageIDs = 2; // Datanodes with replicas of the block + repeated string storageUuids = 2; // Datanodes with replicas of the block } /** From 4551da302d94cffea0313eac79479ab6f9b7cb34 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sun, 22 Sep 2013 18:03:02 +0000 Subject: [PATCH 14/76] HDFS-5233. Use Datanode UUID to identify Datanodes. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1525407 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 + .../apache/hadoop/hdfs/DFSInputStream.java | 2 +- .../hadoop/hdfs/protocol/DatanodeID.java | 66 ++++++++++++------- .../hadoop/hdfs/protocol/DatanodeInfo.java | 6 +- .../protocol/UnregisteredNodeException.java | 2 +- .../hadoop/hdfs/protocolPB/PBHelper.java | 2 +- .../hadoop/hdfs/server/balancer/Balancer.java | 4 +- .../server/blockmanagement/BlockManager.java | 18 ++--- .../blockmanagement/DatanodeManager.java | 24 +++---- .../blockmanagement/InvalidateBlocks.java | 4 +- .../hdfs/server/datanode/BPOfferService.java | 18 ++--- .../hdfs/server/datanode/BPServiceActor.java | 8 +-- .../hdfs/server/datanode/BlockReceiver.java | 2 +- .../hadoop/hdfs/server/datanode/DataNode.java | 18 ++--- .../hdfs/server/datanode/DataXceiver.java | 4 +- .../server/protocol/DatanodeRegistration.java | 2 +- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 2 +- .../hadoop/hdfs/TestDatanodeRegistration.java | 4 +- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 2 +- .../blockmanagement/TestBlockManager.java | 4 +- .../blockmanagement/TestDatanodeManager.java | 6 +- .../server/blockmanagement/TestNodeCount.java | 3 +- .../TestOverReplicatedBlocks.java | 2 +- .../TestPendingReplication.java | 4 +- .../server/datanode/SimulatedFSDataset.java | 13 ++-- .../server/datanode/TestBlockReplacement.java | 2 +- .../hdfs/server/datanode/TestBlockReport.java | 18 ++--- .../TestDataNodeMultipleRegistrations.java | 2 +- .../datanode/TestDataNodeVolumeFailure.java | 2 +- .../namenode/NNThroughputBenchmark.java | 12 ++-- .../server/namenode/TestDeadDatanode.java | 10 +-- 31 files changed, 144 insertions(+), 124 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 9f4e314ad75..348be08ab5c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -28,3 +28,5 @@ IMPROVEMENTS: datanodes. (szetszwo) HDFS-5232. Protocol changes to transmit StorageUuid. (Arpit Agarwal) + + HDFS-5233. Use Datanode UUID to identify Datanodes. (Arpit Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 4131ffa4426..1a31d49b980 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -1290,7 +1290,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead { * deadNodes and added currentNode again. Thats ok. */ deadNodes.remove(oldNode); } - if (!oldNode.getStorageID().equals(newNode.getStorageID())) { + if (!oldNode.getDatanodeUuid().equals(newNode.getDatanodeUuid())) { currentNode = newNode; return true; } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java index 2a0578ca93f..3c1f9f8277c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java @@ -21,6 +21,8 @@ package org.apache.hadoop.hdfs.protocol; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import java.util.UUID; + /** * This class represents the primary identifier for a Datanode. * Datanodes are identified by how they can be contacted (hostname @@ -40,35 +42,42 @@ public class DatanodeID implements Comparable { private String ipAddr; // IP address private String hostName; // hostname claimed by datanode private String peerHostName; // hostname from the actual connection - private String storageID; // unique per cluster storageID private int xferPort; // data streaming port private int infoPort; // info server port private int ipcPort; // IPC server port + // UUID identifying a given datanode. For upgraded Datanodes this is the + // same as the StorageID that was previously used by this Datanode. For + // newly formatted Datanodes it is a UUID. + private String datanodeUuid = null; + public DatanodeID(DatanodeID from) { this(from.getIpAddr(), - from.getHostName(), - from.getStorageID(), - from.getXferPort(), - from.getInfoPort(), - from.getIpcPort()); + from.getHostName(), + from.getDatanodeUuid(), + from.getXferPort(), + from.getInfoPort(), + from.getIpcPort()); this.peerHostName = from.getPeerHostName(); } - + /** * Create a DatanodeID * @param ipAddr IP * @param hostName hostname - * @param storageID data storage ID + * @param datanodeUuid data node ID, UUID for new Datanodes, may be the + * storage ID for pre-UUID datanodes. NULL if unknown + * e.g. if this is a new datanode. A new UUID will + * be assigned by the namenode. * @param xferPort data transfer port * @param infoPort info server port * @param ipcPort ipc server port */ - public DatanodeID(String ipAddr, String hostName, String storageID, - int xferPort, int infoPort, int ipcPort) { + public DatanodeID(String ipAddr, String hostName, String datanodeUuid, + int xferPort, int infoPort, int ipcPort) { this.ipAddr = ipAddr; this.hostName = hostName; - this.storageID = storageID; + this.datanodeUuid = checkDatanodeUuid(datanodeUuid); this.xferPort = xferPort; this.infoPort = infoPort; this.ipcPort = ipcPort; @@ -82,8 +91,28 @@ public class DatanodeID implements Comparable { this.peerHostName = peerHostName; } - public void setStorageID(String storageID) { - this.storageID = storageID; + /** + * @return data node ID. + */ + public String getDatanodeUuid() { + return datanodeUuid; + } + + public void setDatanodeUuid(String datanodeUuid) { + this.datanodeUuid = datanodeUuid; + } + + private String checkDatanodeUuid(String uuid) { + if (uuid == null || uuid.isEmpty()) { + return null; + } else { + return uuid; + } + } + + public String generateNewDatanodeUuid() { + datanodeUuid = UUID.randomUUID().toString(); + return datanodeUuid; } /** @@ -158,13 +187,6 @@ public class DatanodeID implements Comparable { return useHostname ? getIpcAddrWithHostname() : getIpcAddr(); } - /** - * @return data storage ID. - */ - public String getStorageID() { - return storageID; - } - /** * @return xferPort (the port for data streaming) */ @@ -195,12 +217,12 @@ public class DatanodeID implements Comparable { return false; } return (getXferAddr().equals(((DatanodeID)to).getXferAddr()) && - storageID.equals(((DatanodeID)to).getStorageID())); + datanodeUuid.equals(((DatanodeID)to).getDatanodeUuid())); } @Override public int hashCode() { - return getXferAddr().hashCode()^ storageID.hashCode(); + return getXferAddr().hashCode()^ datanodeUuid.hashCode(); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java index 3f5715b0afd..2d5b1bec1e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java @@ -108,18 +108,18 @@ public class DatanodeInfo extends DatanodeID implements Node { final long capacity, final long dfsUsed, final long remaining, final long blockPoolUsed, final long lastUpdate, final int xceiverCount, final AdminStates adminState) { - this(nodeID.getIpAddr(), nodeID.getHostName(), nodeID.getStorageID(), nodeID.getXferPort(), + this(nodeID.getIpAddr(), nodeID.getHostName(), nodeID.getDatanodeUuid(), nodeID.getXferPort(), nodeID.getInfoPort(), nodeID.getIpcPort(), capacity, dfsUsed, remaining, blockPoolUsed, lastUpdate, xceiverCount, location, adminState); } /** Constructor */ public DatanodeInfo(final String ipAddr, final String hostName, - final String storageID, final int xferPort, final int infoPort, final int ipcPort, + final String DatanodeUuid, final int xferPort, final int infoPort, final int ipcPort, final long capacity, final long dfsUsed, final long remaining, final long blockPoolUsed, final long lastUpdate, final int xceiverCount, final String networkLocation, final AdminStates adminState) { - super(ipAddr, hostName, storageID, xferPort, infoPort, ipcPort); + super(ipAddr, hostName, DatanodeUuid, xferPort, infoPort, ipcPort); this.capacity = capacity; this.dfsUsed = dfsUsed; this.remaining = remaining; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java index eabdd22a974..1ef7455539e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnregisteredNodeException.java @@ -51,7 +51,7 @@ public class UnregisteredNodeException extends IOException { */ public UnregisteredNodeException(DatanodeID nodeID, DatanodeInfo storedNode) { super("Data node " + nodeID + " is attempting to report storage ID " - + nodeID.getStorageID() + ". Node " + + nodeID.getDatanodeUuid() + ". Node " + storedNode + " is expected to serve this storage."); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index c2de12e7dce..5c2b5509606 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -234,7 +234,7 @@ public class PBHelper { return DatanodeIDProto.newBuilder() .setIpAddr(dn.getIpAddr()) .setHostName(dn.getHostName()) - .setDatanodeUuid(dn.getStorageID()) + .setDatanodeUuid(dn.getDatanodeUuid()) .setXferPort(dn.getXferPort()) .setInfoPort(dn.getInfoPort()) .setIpcPort(dn.getIpcPort()).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 27cf20a0a07..d512efe11cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -551,7 +551,7 @@ public class Balancer { /* Get the storage id of the datanode */ protected String getStorageID() { - return datanode.getStorageID(); + return datanode.getDatanodeUuid(); } /** Decide if still need to move more bytes */ @@ -895,7 +895,7 @@ public class Balancer { datanodeS.utilization)*datanodeS.datanode.getCapacity()/100.0); } } - this.datanodes.put(datanode.getStorageID(), datanodeS); + this.datanodes.put(datanode.getDatanodeUuid(), datanodeS); } //logging diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 094c791b1df..5684f02da38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -989,7 +989,7 @@ public class BlockManager { } node.resetBlocks(); - invalidateBlocks.remove(node.getStorageID()); + invalidateBlocks.remove(node.getDatanodeUuid()); // If the DN hasn't block-reported since the most recent // failover, then we may have been holding up on processing @@ -1472,7 +1472,7 @@ public class BlockManager { for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) { final DatanodeDescriptor node = storage.getDatanodeDescriptor(); LightWeightLinkedSet excessBlocks = - excessReplicateMap.get(node.getStorageID()); + excessReplicateMap.get(node.getDatanodeUuid()); if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) corrupt++; else if (node.isDecommissionInProgress() || node.isDecommissioned()) @@ -1901,7 +1901,7 @@ public class BlockManager { } // Ignore replicas already scheduled to be removed from the DN - if(invalidateBlocks.contains(dn.getStorageID(), block)) { + if(invalidateBlocks.contains(dn.getDatanodeUuid(), block)) { /* TODO: following assertion is incorrect, see HDFS-2668 assert storedBlock.findDatanode(dn) < 0 : "Block " + block + " in recentInvalidatesSet should not appear in DN " + dn; */ @@ -2441,7 +2441,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block return; } LightWeightLinkedSet excessBlocks = excessReplicateMap.get(cur - .getStorageID()); + .getDatanodeUuid()); if (excessBlocks == null || !excessBlocks.contains(block)) { if (!cur.isDecommissionInProgress() && !cur.isDecommissioned()) { // exclude corrupt replicas @@ -2530,10 +2530,10 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block private void addToExcessReplicate(DatanodeInfo dn, Block block) { assert namesystem.hasWriteLock(); - LightWeightLinkedSet excessBlocks = excessReplicateMap.get(dn.getStorageID()); + LightWeightLinkedSet excessBlocks = excessReplicateMap.get(dn.getDatanodeUuid()); if (excessBlocks == null) { excessBlocks = new LightWeightLinkedSet(); - excessReplicateMap.put(dn.getStorageID(), excessBlocks); + excessReplicateMap.put(dn.getDatanodeUuid(), excessBlocks); } if (excessBlocks.add(block)) { excessBlocksCount.incrementAndGet(); @@ -2581,7 +2581,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block // in "excess" there. // LightWeightLinkedSet excessBlocks = excessReplicateMap.get(node - .getStorageID()); + .getDatanodeUuid()); if (excessBlocks != null) { if (excessBlocks.remove(block)) { excessBlocksCount.decrementAndGet(); @@ -2590,7 +2590,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block + block + " is removed from excessBlocks"); } if (excessBlocks.size() == 0) { - excessReplicateMap.remove(node.getStorageID()); + excessReplicateMap.remove(node.getDatanodeUuid()); } } } @@ -2760,7 +2760,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block decommissioned++; } else { LightWeightLinkedSet blocksExcess = excessReplicateMap.get(node - .getStorageID()); + .getDatanodeUuid()); if (blocksExcess != null && blocksExcess.contains(b)) { excess++; } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index e89dd585553..c468dcb7188 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -418,9 +418,9 @@ public class DatanodeManager { } - /** Get a datanode descriptor given corresponding storageID */ - DatanodeDescriptor getDatanode(final String storageID) { - return datanodeMap.get(storageID); + /** Get a datanode descriptor given corresponding DatanodeUUID */ + DatanodeDescriptor getDatanode(final String datanodeUuid) { + return datanodeMap.get(datanodeUuid); } /** @@ -432,7 +432,7 @@ public class DatanodeManager { */ public DatanodeDescriptor getDatanode(DatanodeID nodeID ) throws UnregisteredNodeException { - final DatanodeDescriptor node = getDatanode(nodeID.getStorageID()); + final DatanodeDescriptor node = getDatanode(nodeID.getDatanodeUuid()); if (node == null) return null; if (!node.getXferAddr().equals(nodeID.getXferAddr())) { @@ -536,7 +536,7 @@ public class DatanodeManager { // remove from host2DatanodeMap the datanodeDescriptor removed // from datanodeMap before adding node to host2DatanodeMap. synchronized(datanodeMap) { - host2DatanodeMap.remove(datanodeMap.put(node.getStorageID(), node)); + host2DatanodeMap.remove(datanodeMap.put(node.getDatanodeUuid(), node)); } networktopology.add(node); // may throw InvalidTopologyException @@ -551,7 +551,7 @@ public class DatanodeManager { /** Physically remove node from datanodeMap. */ private void wipeDatanode(final DatanodeID node) { - final String key = node.getStorageID(); + final String key = node.getDatanodeUuid(); synchronized (datanodeMap) { host2DatanodeMap.remove(datanodeMap.remove(key)); } @@ -774,9 +774,9 @@ public class DatanodeManager { } NameNode.stateChangeLog.info("BLOCK* registerDatanode: from " - + nodeReg + " storage " + nodeReg.getStorageID()); + + nodeReg + " storage " + nodeReg.getDatanodeUuid()); - DatanodeDescriptor nodeS = datanodeMap.get(nodeReg.getStorageID()); + DatanodeDescriptor nodeS = datanodeMap.get(nodeReg.getDatanodeUuid()); DatanodeDescriptor nodeN = host2DatanodeMap.getDatanodeByXferAddr( nodeReg.getIpAddr(), nodeReg.getXferPort()); @@ -811,7 +811,7 @@ public class DatanodeManager { */ NameNode.stateChangeLog.info("BLOCK* registerDatanode: " + nodeS + " is replaced by " + nodeReg + " with the same storageID " - + nodeReg.getStorageID()); + + nodeReg.getDatanodeUuid()); } boolean success = false; @@ -846,14 +846,14 @@ public class DatanodeManager { } // this is a new datanode serving a new data storage - if ("".equals(nodeReg.getStorageID())) { + if ("".equals(nodeReg.getDatanodeUuid())) { // this data storage has never been registered // it is either empty or was created by pre-storageID version of DFS - nodeReg.setStorageID(DatanodeStorage.newStorageID()); + nodeReg.setDatanodeUuid(DatanodeStorage.newStorageID()); if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug( "BLOCK* NameSystem.registerDatanode: " - + "new storageID " + nodeReg.getStorageID() + " assigned."); + + "new Datanode UUID " + nodeReg.getDatanodeUuid() + " assigned."); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java index 841ca41755f..06eea7b9708 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java @@ -78,10 +78,10 @@ class InvalidateBlocks { */ synchronized void add(final Block block, final DatanodeInfo datanode, final boolean log) { - LightWeightHashSet set = node2blocks.get(datanode.getStorageID()); + LightWeightHashSet set = node2blocks.get(datanode.getDatanodeUuid()); if (set == null) { set = new LightWeightHashSet(); - node2blocks.put(datanode.getStorageID(), set); + node2blocks.put(datanode.getDatanodeUuid(), set); } if (set.add(block)) { numBlocks++; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index d5e026b33b1..e11cc87b4a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -159,24 +159,24 @@ class BPOfferService { synchronized NamespaceInfo getNamespaceInfo() { return bpNSInfo; } - + @Override public String toString() { if (bpNSInfo == null) { // If we haven't yet connected to our NN, we don't yet know our // own block pool ID. // If _none_ of the block pools have connected yet, we don't even - // know the storage ID of this DN. - String storageId = dn.getStorageId(); - if (storageId == null || "".equals(storageId)) { - storageId = "unknown"; + // know the DatanodeID ID of this DN. + String datanodeUuid = dn.getDatanodeUuid(); + + if (datanodeUuid == null || datanodeUuid.isEmpty()) { + datanodeUuid = "unassigned"; } - return "Block pool (storage id " + storageId + - ")"; + return "Block pool (Datanode Uuid " + datanodeUuid + ")"; } else { return "Block pool " + getBlockPoolId() + - " (storage id " + dn.getStorageId() + - ")"; + " (Datanode Uuid " + dn.getDatanodeUuid() + + ")"; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index e2a688af85a..4d61cdfb447 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -22,7 +22,6 @@ import static org.apache.hadoop.util.Time.now; import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketTimeoutException; -import java.net.URI; import java.util.Collection; import java.util.Map; @@ -50,7 +49,6 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.util.VersionUtil; @@ -279,7 +277,7 @@ class BPServiceActor implements Runnable { } if (receivedAndDeletedBlockArray != null) { StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( - bpRegistration.getStorageID(), receivedAndDeletedBlockArray) }; + bpRegistration.getDatanodeUuid(), receivedAndDeletedBlockArray) }; boolean success = false; try { bpNamenode.blockReceivedAndDeleted(bpRegistration, bpos.getBlockPoolId(), @@ -398,7 +396,7 @@ class BPServiceActor implements Runnable { // Send block report long brSendStartTime = now(); StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(bpRegistration.getStorageID()), + new DatanodeStorage(bpRegistration.getDatanodeUuid()), bReport.getBlockListAsLongs()) }; cmd = bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), report); @@ -436,7 +434,7 @@ class BPServiceActor implements Runnable { LOG.debug("Sending heartbeat from service actor: " + this); } // reports number of failed volumes - StorageReport[] report = { new StorageReport(bpRegistration.getStorageID(), + StorageReport[] report = { new StorageReport(bpRegistration.getDatanodeUuid(), false, dn.getFSDataset().getCapacity(), dn.getFSDataset().getDfsUsed(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index e3f7499271d..625daf06253 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -1079,7 +1079,7 @@ class BlockReceiver implements Closeable { .getBlockPoolId()); ClientTraceLog.info(String.format(DN_CLIENTTRACE_FORMAT, inAddr, myAddr, block.getNumBytes(), "HDFS_WRITE", clientname, offset, - dnR.getStorageID(), block, endTime - startTime)); + dnR.getDatanodeUuid(), block, endTime - startTime)); } else { LOG.info("Received " + block + " size " + block.getNumBytes() + " from " + inAddr); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 0769d9c657e..d541c6ff3e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -750,7 +750,7 @@ public class DataNode extends Configured } DatanodeID dnId = new DatanodeID( streamingAddr.getAddress().getHostAddress(), hostName, - getStorageId(), getXferPort(), getInfoPort(), getIpcPort()); + getDatanodeUuid(), getXferPort(), getInfoPort(), getIpcPort()); return new DatanodeRegistration(dnId, storageInfo, new ExportedBlockKeys(), VersionInfo.getVersion()); } @@ -770,13 +770,13 @@ public class DataNode extends Configured if (storage.getStorageID().equals("")) { // This is a fresh datanode, persist the NN-provided storage ID - storage.setStorageID(bpRegistration.getStorageID()); + storage.setStorageID(bpRegistration.getDatanodeUuid()); storage.writeAll(); - LOG.info("New storage id " + bpRegistration.getStorageID() + LOG.info("New storage id " + bpRegistration.getDatanodeUuid() + " is assigned to data-node " + bpRegistration); - } else if(!storage.getStorageID().equals(bpRegistration.getStorageID())) { + } else if(!storage.getStorageID().equals(bpRegistration.getDatanodeUuid())) { throw new IOException("Inconsistent storage IDs. Name-node returned " - + bpRegistration.getStorageID() + + bpRegistration.getDatanodeUuid() + ". Expecting " + storage.getStorageID()); } @@ -925,7 +925,7 @@ public class DataNode extends Configured return streamingAddr.getPort(); } - String getStorageId() { + String getDatanodeUuid() { return storage.getStorageID(); } @@ -940,7 +940,7 @@ public class DataNode extends Configured /** * NB: The datanode can perform data transfer on the streaming * address however clients are given the IPC IP address for data - * transfer, and that may be a different address. + * transfer, and that may be a different address. * * @return socket address for data transfer */ @@ -1016,7 +1016,7 @@ public class DataNode extends Configured public static void setNewStorageID(DatanodeID dnId) { LOG.info("Datanode is " + dnId); - dnId.setStorageID(DatanodeStorage.newStorageID()); + dnId.setDatanodeUuid(DatanodeStorage.newStorageID()); } /** Ensure the authentication method is kerberos */ @@ -1818,7 +1818,7 @@ public class DataNode extends Configured @Override public String toString() { return "DataNode{data=" + data + ", localName='" + getDisplayName() - + "', storageID='" + getStorageId() + "', xmitsInProgress=" + + "', storageID='" + getDatanodeUuid() + "', xmitsInProgress=" + xmitsInProgress.get() + "}"; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index a112845cbaf..e1de714d2ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -284,7 +284,7 @@ class DataXceiver extends Receiver implements Runnable { BlockSender.ClientTraceLog.info(String.format( "src: 127.0.0.1, dest: 127.0.0.1, op: REQUEST_SHORT_CIRCUIT_FDS," + " blockid: %s, srvID: %s, success: %b", - blk.getBlockId(), dnR.getStorageID(), (fis != null) + blk.getBlockId(), dnR.getDatanodeUuid(), (fis != null) )); } if (fis != null) { @@ -317,7 +317,7 @@ class DataXceiver extends Receiver implements Runnable { clientName.length() > 0 && ClientTraceLog.isInfoEnabled() ? String.format(DN_CLIENTTRACE_FORMAT, localAddress, remoteAddress, "%d", "HDFS_READ", clientName, "%d", - dnR.getStorageID(), block, "%d") + dnR.getDatanodeUuid(), block, "%d") : dnR + " Served block " + block + " to " + remoteAddress; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java index 7815d808d62..c4c30f8bb8e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java @@ -82,7 +82,7 @@ public class DatanodeRegistration extends DatanodeID public String toString() { return getClass().getSimpleName() + "(" + getIpAddr() - + ", storageID=" + getStorageID() + + ", storageID=" + getDatanodeUuid() + ", infoPort=" + getInfoPort() + ", ipcPort=" + getIpcPort() + ", storageInfo=" + storageInfo diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 840087393cd..268d344f5ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -292,7 +292,7 @@ public class JsonUtil { final Map m = new TreeMap(); m.put("ipAddr", datanodeinfo.getIpAddr()); m.put("hostName", datanodeinfo.getHostName()); - m.put("storageID", datanodeinfo.getStorageID()); + m.put("storageID", datanodeinfo.getDatanodeUuid()); m.put("xferPort", datanodeinfo.getXferPort()); m.put("infoPort", datanodeinfo.getInfoPort()); m.put("ipcPort", datanodeinfo.getIpcPort()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java index a94d1f4dd2c..e465ed3b8e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java @@ -223,7 +223,7 @@ public class TestDatanodeRegistration { DatanodeRegistration mockDnReg = mock(DatanodeRegistration.class); doReturn(HdfsConstants.LAYOUT_VERSION).when(mockDnReg).getVersion(); doReturn(123).when(mockDnReg).getXferPort(); - doReturn("fake-storage-id").when(mockDnReg).getStorageID(); + doReturn("fake-storage-id").when(mockDnReg).getDatanodeUuid(); doReturn(mockStorageInfo).when(mockDnReg).getStorageInfo(); // Should succeed when software versions are the same. @@ -270,7 +270,7 @@ public class TestDatanodeRegistration { DatanodeRegistration mockDnReg = mock(DatanodeRegistration.class); doReturn(HdfsConstants.LAYOUT_VERSION).when(mockDnReg).getVersion(); - doReturn("fake-storage-id").when(mockDnReg).getStorageID(); + doReturn("fake-storage-id").when(mockDnReg).getDatanodeUuid(); doReturn(mockStorageInfo).when(mockDnReg).getStorageInfo(); // Should succeed when software versions are the same and CTimes are the diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index d8d4329d1b9..2b1817956f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -145,7 +145,7 @@ public class TestPBHelper { void compare(DatanodeID dn, DatanodeID dn2) { assertEquals(dn.getIpAddr(), dn2.getIpAddr()); assertEquals(dn.getHostName(), dn2.getHostName()); - assertEquals(dn.getStorageID(), dn2.getStorageID()); + assertEquals(dn.getDatanodeUuid(), dn2.getDatanodeUuid()); assertEquals(dn.getXferPort(), dn2.getXferPort()); assertEquals(dn.getInfoPort(), dn2.getInfoPort()); assertEquals(dn.getIpcPort(), dn2.getIpcPort()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index f1b2cf77fef..e0d5698f76f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -527,7 +527,7 @@ public class TestBlockManager { public void testSafeModeIBR() throws Exception { DatanodeDescriptor node = spy(nodes.get(0)); DatanodeStorageInfo ds = node.getStorageInfos()[0]; - node.setStorageID(ds.getStorageID()); + node.setDatanodeUuid(ds.getStorageID()); node.isAlive = true; @@ -571,7 +571,7 @@ public class TestBlockManager { public void testSafeModeIBRAfterIncremental() throws Exception { DatanodeDescriptor node = spy(nodes.get(0)); DatanodeStorageInfo ds = node.getStorageInfos()[0]; - node.setStorageID(ds.getStorageID()); + node.setDatanodeUuid(ds.getStorageID()); node.isAlive = true; DatanodeRegistration nodeReg = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java index 75034932acd..84d0d6b5800 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java @@ -76,7 +76,7 @@ public class TestDatanodeManager { it.next(); } DatanodeRegistration toRemove = it.next().getValue(); - Log.info("Removing node " + toRemove.getStorageID() + " ip " + + Log.info("Removing node " + toRemove.getDatanodeUuid() + " ip " + toRemove.getXferAddr() + " version : " + toRemove.getSoftwareVersion()); //Remove that random node @@ -90,7 +90,7 @@ public class TestDatanodeManager { String storageID = "someStorageID" + rng.nextInt(5000); DatanodeRegistration dr = Mockito.mock(DatanodeRegistration.class); - Mockito.when(dr.getStorageID()).thenReturn(storageID); + Mockito.when(dr.getDatanodeUuid()).thenReturn(storageID); //If this storageID had already been registered before if(sIdToDnReg.containsKey(storageID)) { @@ -110,7 +110,7 @@ public class TestDatanodeManager { Mockito.when(dr.getSoftwareVersion()).thenReturn( "version" + rng.nextInt(5)); - Log.info("Registering node storageID: " + dr.getStorageID() + + Log.info("Registering node storageID: " + dr.getDatanodeUuid() + ", version: " + dr.getSoftwareVersion() + ", IP address: " + dr.getXferAddr()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java index d66c9254e5c..f7973819ecb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import static org.junit.Assert.assertTrue; import java.util.Collection; -import java.util.Iterator; import java.util.concurrent.TimeoutException; import org.apache.hadoop.conf.Configuration; @@ -101,7 +100,7 @@ public class TestNodeCount { DatanodeDescriptor nonExcessDN = null; for(DatanodeStorageInfo storage : bm.blocksMap.getStorages(block.getLocalBlock())) { final DatanodeDescriptor dn = storage.getDatanodeDescriptor(); - Collection blocks = bm.excessReplicateMap.get(dn.getStorageID()); + Collection blocks = bm.excessReplicateMap.get(dn.getDatanodeUuid()); if (blocks == null || !blocks.contains(block.getLocalBlock()) ) { nonExcessDN = dn; break; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java index 972a785f5d6..412dd611de5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java @@ -155,7 +155,7 @@ public class TestOverReplicatedBlocks { DataNode lastDN = cluster.getDataNodes().get(3); DatanodeRegistration dnReg = DataNodeTestUtils.getDNRegistrationForBP( lastDN, namesystem.getBlockPoolId()); - String lastDNid = dnReg.getStorageID(); + String lastDNid = dnReg.getDatanodeUuid(); final Path fileName = new Path("/foo2"); DFSTestUtil.createFile(fs, fileName, SMALL_FILE_LENGTH, (short)4, 0L); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java index 74aef6d6a71..71da2b3c22d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java @@ -198,7 +198,7 @@ public class TestPendingReplication { DatanodeRegistration dnR = datanodes.get(i).getDNRegistrationForBP( poolId); StorageReceivedDeletedBlocks[] report = { - new StorageReceivedDeletedBlocks(dnR.getStorageID(), + new StorageReceivedDeletedBlocks(dnR.getDatanodeUuid(), new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo( blocks[0], BlockStatus.RECEIVED_BLOCK, "") }) }; cluster.getNameNodeRpc().blockReceivedAndDeleted(dnR, poolId, report); @@ -215,7 +215,7 @@ public class TestPendingReplication { DatanodeRegistration dnR = datanodes.get(i).getDNRegistrationForBP( poolId); StorageReceivedDeletedBlocks[] report = - { new StorageReceivedDeletedBlocks(dnR.getStorageID(), + { new StorageReceivedDeletedBlocks(dnR.getDatanodeUuid(), new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo( blocks[0], BlockStatus.RECEIVED_BLOCK, "") }) }; cluster.getNameNodeRpc().blockReceivedAndDeleted(dnR, poolId, report); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 800a628a4e9..1f24375a0ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -380,16 +379,16 @@ public class SimulatedFSDataset implements FsDatasetSpi { private final Map> blockMap = new HashMap>(); private final SimulatedStorage storage; - private final String storageId; + private final String datanodeUuid; public SimulatedFSDataset(DataStorage storage, Configuration conf) { if (storage != null) { storage.createStorageID(); - this.storageId = storage.getStorageID(); + this.datanodeUuid = storage.getStorageID(); } else { - this.storageId = "unknownStorageId-" + UUID.randomUUID(); + this.datanodeUuid = "unknownStorageId-" + UUID.randomUUID(); } - registerMBean(storageId); + registerMBean(datanodeUuid); this.storage = new SimulatedStorage( conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY)); } @@ -884,7 +883,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { @Override public String getStorageInfo() { - return "Simulated FSDataset-" + storageId; + return "Simulated FSDataset-" + datanodeUuid; } @Override @@ -911,7 +910,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { public String updateReplicaUnderRecovery(ExtendedBlock oldBlock, long recoveryId, long newlength) { - return storageId; + return datanodeUuid; } @Override // FsDatasetSpi diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java index a138071b525..02faa595e4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java @@ -265,7 +265,7 @@ public class TestBlockReplacement { // sendRequest DataOutputStream out = new DataOutputStream(sock.getOutputStream()); new Sender(out).replaceBlock(block, BlockTokenSecretManager.DUMMY_TOKEN, - source.getStorageID(), sourceProxy); + source.getDatanodeUuid(), sourceProxy); out.flush(); // receiveResponse DataInputStream reply = new DataInputStream(sock.getInputStream()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java index d1c3d9eccb1..1bc8db1e253 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java @@ -154,7 +154,7 @@ public class TestBlockReport { String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), + new DatanodeStorage(dnR.getDatanodeUuid()), new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; cluster.getNameNodeRpc().blockReport(dnR, poolId, report); @@ -236,7 +236,7 @@ public class TestBlockReport { String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn0.getDNRegistrationForBP(poolId); StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), + new DatanodeStorage(dnR.getDatanodeUuid()), new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; cluster.getNameNodeRpc().blockReport(dnR, poolId, report); @@ -278,7 +278,7 @@ public class TestBlockReport { String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), + new DatanodeStorage(dnR.getDatanodeUuid()), new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; DatanodeCommand dnCmd = cluster.getNameNodeRpc().blockReport(dnR, poolId, report); @@ -332,7 +332,7 @@ public class TestBlockReport { String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), + new DatanodeStorage(dnR.getDatanodeUuid()), new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); @@ -368,7 +368,7 @@ public class TestBlockReport { int randIndex = rand.nextInt(blocks.size()); // Get a block and screw its GS Block corruptedBlock = blocks.get(randIndex); - String secondNode = cluster.getDataNodes().get(DN_N1).getStorageId(); + String secondNode = cluster.getDataNodes().get(DN_N1).getDatanodeUuid(); if(LOG.isDebugEnabled()) { LOG.debug("Working with " + secondNode); LOG.debug("BlockGS before " + blocks.get(randIndex).getGenerationStamp()); @@ -383,7 +383,7 @@ public class TestBlockReport { String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), + new DatanodeStorage(dnR.getDatanodeUuid()), new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); @@ -407,7 +407,7 @@ public class TestBlockReport { } report[0] = new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), + new DatanodeStorage(dnR.getDatanodeUuid()), new BlockListAsLongs(blocks, null).getBlockListAsLongs()); cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); @@ -459,7 +459,7 @@ public class TestBlockReport { String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), + new DatanodeStorage(dnR.getDatanodeUuid()), new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); @@ -507,7 +507,7 @@ public class TestBlockReport { String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), + new DatanodeStorage(dnR.getDatanodeUuid()), new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java index 50272f1dda2..8ce8569df41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java @@ -163,7 +163,7 @@ public class TestDataNodeMultipleRegistrations { for (BPOfferService bpos : dn.getAllBpOs()) { LOG.info("reg: bpid=" + "; name=" + bpos.bpRegistration + "; sid=" - + bpos.bpRegistration.getStorageID() + "; nna=" + + + bpos.bpRegistration.getDatanodeUuid() + "; nna=" + getNNSocketAddress(bpos)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java index 75482ae4426..706916d8705 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java @@ -153,7 +153,7 @@ public class TestDataNodeVolumeFailure { DatanodeRegistration dnR = dn.getDNRegistrationForBP(bpid); final StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getStorageID()), + new DatanodeStorage(dnR.getDatanodeUuid()), DataNodeTestUtils.getFSDataset(dn).getBlockReport(bpid ).getBlockListAsLongs()) }; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java index 3156de4e93a..f88015c2954 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java @@ -846,7 +846,7 @@ public class NNThroughputBenchmark implements Tool { // register datanode dnRegistration = nameNodeProto.registerDatanode(dnRegistration); //first block reports - storage = new DatanodeStorage(dnRegistration.getStorageID()); + storage = new DatanodeStorage(dnRegistration.getDatanodeUuid()); final StorageBlockReport[] reports = { new StorageBlockReport(storage, new BlockListAsLongs(null, null).getBlockListAsLongs()) @@ -862,7 +862,7 @@ public class NNThroughputBenchmark implements Tool { void sendHeartbeat() throws IOException { // register datanode // TODO:FEDERATION currently a single block pool is supported - StorageReport[] rep = { new StorageReport(dnRegistration.getStorageID(), + StorageReport[] rep = { new StorageReport(dnRegistration.getDatanodeUuid(), false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) }; DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep, 0, 0, 0).getCommands(); @@ -909,7 +909,7 @@ public class NNThroughputBenchmark implements Tool { @SuppressWarnings("unused") // keep it for future blockReceived benchmark int replicateBlocks() throws IOException { // register datanode - StorageReport[] rep = { new StorageReport(dnRegistration.getStorageID(), + StorageReport[] rep = { new StorageReport(dnRegistration.getDatanodeUuid(), false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) }; DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep, 0, 0, 0).getCommands(); @@ -939,14 +939,14 @@ public class NNThroughputBenchmark implements Tool { DatanodeInfo dnInfo = blockTargets[t]; DatanodeRegistration receivedDNReg; receivedDNReg = new DatanodeRegistration(dnInfo, - new DataStorage(nsInfo, dnInfo.getStorageID()), + new DataStorage(nsInfo, dnInfo.getDatanodeUuid()), new ExportedBlockKeys(), VersionInfo.getVersion()); ReceivedDeletedBlockInfo[] rdBlocks = { new ReceivedDeletedBlockInfo( blocks[i], ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( - receivedDNReg.getStorageID(), rdBlocks) }; + receivedDNReg.getDatanodeUuid(), rdBlocks) }; nameNodeProto.blockReceivedAndDeleted(receivedDNReg, nameNode .getNamesystem().getBlockPoolId(), report); } @@ -1069,7 +1069,7 @@ public class NNThroughputBenchmark implements Tool { loc.getBlock().getLocalBlock(), ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( - datanodes[dnIdx].dnRegistration.getStorageID(), rdBlocks) }; + datanodes[dnIdx].dnRegistration.getDatanodeUuid(), rdBlocks) }; nameNodeProto.blockReceivedAndDeleted(datanodes[dnIdx].dnRegistration, loc .getBlock().getBlockPoolId(), report); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java index d78198ab402..5461a09c260 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java @@ -104,11 +104,11 @@ public class TestDeadDatanode { DatanodeRegistration reg = DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId); - waitForDatanodeState(reg.getStorageID(), true, 20000); + waitForDatanodeState(reg.getDatanodeUuid(), true, 20000); // Shutdown and wait for datanode to be marked dead dn.shutdown(); - waitForDatanodeState(reg.getStorageID(), false, 20000); + waitForDatanodeState(reg.getDatanodeUuid(), false, 20000); DatanodeProtocol dnp = cluster.getNameNodeRpc(); @@ -117,7 +117,7 @@ public class TestDeadDatanode { ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; StorageReceivedDeletedBlocks[] storageBlocks = { - new StorageReceivedDeletedBlocks(reg.getStorageID(), blocks) }; + new StorageReceivedDeletedBlocks(reg.getDatanodeUuid(), blocks) }; // Ensure blockReceived call from dead datanode is rejected with IOException try { @@ -129,7 +129,7 @@ public class TestDeadDatanode { // Ensure blockReport from dead datanode is rejected with IOException StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(reg.getStorageID()), + new DatanodeStorage(reg.getDatanodeUuid()), new long[] { 0L, 0L, 0L }) }; try { dnp.blockReport(reg, poolId, report); @@ -140,7 +140,7 @@ public class TestDeadDatanode { // Ensure heartbeat from dead datanode is rejected with a command // that asks datanode to register again - StorageReport[] rep = { new StorageReport(reg.getStorageID(), false, 0, 0, + StorageReport[] rep = { new StorageReport(reg.getDatanodeUuid(), false, 0, 0, 0, 0) }; DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, rep, 0, 0, 0).getCommands(); assertEquals(1, cmd.length); From c6d09d8c0c1801d41f3d8450f9836115aca7d1a3 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 25 Sep 2013 16:03:20 +0000 Subject: [PATCH 15/76] HDFS-5222. Move block schedule information from DatanodeDescriptor to DatanodeStorageInfo. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1526215 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 + .../server/blockmanagement/BlockManager.java | 9 +-- .../BlockPlacementPolicyDefault.java | 13 +--- .../blockmanagement/DatanodeDescriptor.java | 47 ++------------ .../blockmanagement/DatanodeStorageInfo.java | 62 +++++++++++++++++-- .../hdfs/server/namenode/FSNamesystem.java | 4 +- 6 files changed, 72 insertions(+), 66 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 348be08ab5c..b99955c4e2f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -30,3 +30,6 @@ IMPROVEMENTS: HDFS-5232. Protocol changes to transmit StorageUuid. (Arpit Agarwal) HDFS-5233. Use Datanode UUID to identify Datanodes. (Arpit Agarwal) + + HDFS-5222. Move block schedule information from DatanodeDescriptor to + DatanodeStorageInfo. (szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 5684f02da38..5d5d02dce25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1328,10 +1328,7 @@ public class BlockManager { // Add block to the to be replicated list rw.srcNode.addBlockToBeReplicated(block, targets); scheduledWork++; - - for (DatanodeStorageInfo storage : targets) { - storage.getDatanodeDescriptor().incBlocksScheduled(); - } + DatanodeStorageInfo.incrementBlocksScheduled(targets); // Move the block-replication into a "pending" state. // The reason we use 'pending' is so we can retry @@ -2621,10 +2618,10 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block @VisibleForTesting void addBlock(DatanodeDescriptor node, String storageID, Block block, String delHint) throws IOException { - // decrement number of blocks scheduled to this datanode. + // Decrement number of blocks scheduled to this storage. // for a retry request (of DatanodeProtocol#blockReceivedAndDeleted with // RECEIVED_BLOCK), we currently also decrease the approximate number. - node.decBlocksScheduled(); + node.getStorageInfo(storageID).decrementBlocksScheduled(); // get the deletion hint node DatanodeDescriptor delHintNode = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index af786e716dc..493e6f87c26 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -620,19 +620,12 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { } final long requiredSize = blockSize * HdfsConstants.MIN_BLOCKS_FOR_WRITE; - if (requiredSize > storage.getRemaining()) { + final long scheduledSize = blockSize = storage.getBlocksScheduled(); + if (requiredSize > storage.getRemaining() - scheduledSize) { logNodeIsNotChosen(storage, "the storage does not have enough space "); return false; } - //TODO: move getBlocksScheduled() to DatanodeStorageInfo. - long remaining = node.getRemaining() - - (node.getBlocksScheduled() * blockSize); - // check the remaining capacity of the target machine - if (requiredSize > remaining) { - logNodeIsNotChosen(storage, "the node does not have enough space "); - return false; - } - + // check the communication traffic of the target machine if (considerLoad) { double avgLoad = 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 4cdac91faf6..d2343bcc400 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -125,15 +125,6 @@ public class DatanodeDescriptor extends DatanodeInfo { /** A set of blocks to be invalidated by this datanode */ private LightWeightHashSet invalidateBlocks = new LightWeightHashSet(); - /* Variables for maintaining number of blocks scheduled to be written to - * this datanode. This count is approximate and might be slightly bigger - * in case of errors (e.g. datanode does not report if an error occurs - * while writing the block). - */ - private int currApproxBlocksScheduled = 0; - private int prevApproxBlocksScheduled = 0; - private long lastBlocksScheduledRollTime = 0; - private static final int BLOCKS_SCHEDULED_ROLL_INTERVAL = 600*1000; //10min private int volumeFailures = 0; /** @@ -313,9 +304,8 @@ public class DatanodeDescriptor extends DatanodeInfo { setLastUpdate(Time.now()); this.volumeFailures = volFailures; for(DatanodeStorageInfo storage : getStorageInfos()) { - storage.receivedHeartbeat(); + storage.receivedHeartbeat(getLastUpdate()); } - rollBlocksScheduled(getLastUpdate()); } private static class BlockIterator implements Iterator { @@ -437,38 +427,11 @@ public class DatanodeDescriptor extends DatanodeInfo { * to this datanode. */ public int getBlocksScheduled() { - return currApproxBlocksScheduled + prevApproxBlocksScheduled; - } - - /** - * Increments counter for number of blocks scheduled. - */ - public void incBlocksScheduled() { - currApproxBlocksScheduled++; - } - - /** - * Decrements counter for number of blocks scheduled. - */ - void decBlocksScheduled() { - if (prevApproxBlocksScheduled > 0) { - prevApproxBlocksScheduled--; - } else if (currApproxBlocksScheduled > 0) { - currApproxBlocksScheduled--; - } - // its ok if both counters are zero. - } - - /** - * Adjusts curr and prev number of blocks scheduled every few minutes. - */ - private void rollBlocksScheduled(long now) { - if ((now - lastBlocksScheduledRollTime) > - BLOCKS_SCHEDULED_ROLL_INTERVAL) { - prevApproxBlocksScheduled = currApproxBlocksScheduled; - currApproxBlocksScheduled = 0; - lastBlocksScheduledRollTime = now; + int n = 0; + for(DatanodeStorageInfo storage : getStorageInfos()) { + n += storage.getBlocksScheduled(); } + return n; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index 6695e072446..af07e9c2c17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -92,9 +92,11 @@ public class DatanodeStorageInfo { private final String storageID; private final StorageType storageType; private State state; + private long capacity; private long dfsUsed; private long remaining; + private volatile BlockInfo blockList = null; private int numBlocks = 0; @@ -117,6 +119,16 @@ public class DatanodeStorageInfo { */ private boolean blockContentsStale = true; + /* Variables for maintaining number of blocks scheduled to be written to + * this storage. This count is approximate and might be slightly bigger + * in case of errors (e.g. datanode does not report if an error occurs + * while writing the block). + */ + private int currApproxBlocksScheduled = 0; + private int prevApproxBlocksScheduled = 0; + private long lastBlocksScheduledRollTime = 0; + private static final int BLOCKS_SCHEDULED_ROLL_INTERVAL = 600*1000; //10min + public DatanodeStorageInfo(DatanodeDescriptor dn, DatanodeStorage s) { this.dn = dn; this.storageID = s.getStorageID(); @@ -132,27 +144,28 @@ public class DatanodeStorageInfo { this.blockReportCount = blockReportCount; } - public boolean areBlockContentsStale() { + boolean areBlockContentsStale() { return blockContentsStale; } - public void markStaleAfterFailover() { + void markStaleAfterFailover() { heartbeatedSinceFailover = false; blockContentsStale = true; } - public void receivedHeartbeat() { + void receivedHeartbeat(final long lastUpdate) { heartbeatedSinceFailover = true; + rollBlocksScheduled(lastUpdate); } - public void receivedBlockReport() { + void receivedBlockReport() { if (heartbeatedSinceFailover) { blockContentsStale = false; } blockReportCount++; } - public void setUtilization(long capacity, long dfsUsed, long remaining) { + void setUtilization(long capacity, long dfsUsed, long remaining) { this.capacity = capacity; this.dfsUsed = dfsUsed; this.remaining = remaining; @@ -224,7 +237,46 @@ public class DatanodeStorageInfo { public DatanodeDescriptor getDatanodeDescriptor() { return dn; } + + /** + * @return Approximate number of blocks currently scheduled to be written + * to this storage. + */ + int getBlocksScheduled() { + return currApproxBlocksScheduled + prevApproxBlocksScheduled; + } + + /** Increment the number of blocks scheduled for each given storage */ + public static void incrementBlocksScheduled(DatanodeStorageInfo... storages) { + for (DatanodeStorageInfo s : storages) { + s.incrementBlocksScheduled(); + } + } + + /** Increment the number of blocks scheduled. */ + private void incrementBlocksScheduled() { + currApproxBlocksScheduled++; + } + /** Decrement the number of blocks scheduled. */ + void decrementBlocksScheduled() { + if (prevApproxBlocksScheduled > 0) { + prevApproxBlocksScheduled--; + } else if (currApproxBlocksScheduled > 0) { + currApproxBlocksScheduled--; + } + // its ok if both counters are zero. + } + + /** Adjusts curr and prev number of blocks scheduled every few minutes. */ + private void rollBlocksScheduled(long now) { + if (now - lastBlocksScheduledRollTime > BLOCKS_SCHEDULED_ROLL_INTERVAL) { + prevApproxBlocksScheduled = currApproxBlocksScheduled; + currApproxBlocksScheduled = 0; + lastBlocksScheduledRollTime = now; + } + } + @Override public boolean equals(Object obj) { if (this == obj) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index ebf69dff20e..38aef7ca7c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2868,9 +2868,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, BlockInfo b = dir.addBlock(src, inodesInPath, newBlock, targets); NameNode.stateChangeLog.info("BLOCK* allocateBlock: " + src + ". " + getBlockPoolId() + " " + b); - for (DatanodeStorageInfo storage : targets) { - storage.getDatanodeDescriptor().incBlocksScheduled(); - } + DatanodeStorageInfo.incrementBlocksScheduled(targets); return b; } From 46099ce7f1a1d5aab85d9408dc1454fcbe54f7e8 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 27 Sep 2013 16:05:39 +0000 Subject: [PATCH 16/76] HDFS-4988. Datanode must support all the volumes as individual storages. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1526969 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 + .../hadoop/hdfs/protocol/LayoutVersion.java | 4 +- .../hadoop/hdfs/protocolPB/PBHelper.java | 5 +- .../blockmanagement/DatanodeManager.java | 20 ++-- .../hadoop/hdfs/server/common/Storage.java | 11 +++ .../hdfs/server/datanode/BPServiceActor.java | 33 +++++-- .../hadoop/hdfs/server/datanode/DataNode.java | 36 +++---- .../hdfs/server/datanode/DataStorage.java | 94 +++++++++++------- .../datanode/fsdataset/FsDatasetSpi.java | 11 ++- .../fsdataset/impl/FsDatasetImpl.java | 98 ++++++++++++------- .../datanode/fsdataset/impl/FsVolumeImpl.java | 2 +- .../datanode/fsdataset/impl/FsVolumeList.java | 35 ++++--- .../datanode/fsdataset/impl/ReplicaMap.java | 9 ++ .../org/apache/hadoop/hdfs/web/JsonUtil.java | 2 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 23 ++--- .../hadoop/hdfs/TestDatanodeRegistration.java | 4 +- .../org/apache/hadoop/hdfs/TestPeerCache.java | 10 +- .../apache/hadoop/hdfs/UpgradeUtilities.java | 2 +- .../blockmanagement/TestBlockManager.java | 5 + .../hdfs/server/common/TestJspHelper.java | 4 +- .../server/datanode/SimulatedFSDataset.java | 21 ++-- .../hdfs/server/datanode/TestBlockReport.java | 2 +- .../namenode/NNThroughputBenchmark.java | 6 +- 23 files changed, 283 insertions(+), 157 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index b99955c4e2f..3b975f242b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -33,3 +33,6 @@ IMPROVEMENTS: HDFS-5222. Move block schedule information from DatanodeDescriptor to DatanodeStorageInfo. (szetszwo) + + HDFS-4988. Datanode must support all the volumes as individual storages. + (Arpit Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java index d4c62c4c710..2cb5b7698a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java @@ -106,7 +106,9 @@ public class LayoutVersion { SEQUENTIAL_BLOCK_ID(-46, "Allocate block IDs sequentially and store " + "block IDs in the edits log and image files"), EDITLOG_SUPPORT_RETRYCACHE(-47, "Record ClientId and CallId in editlog to " - + "enable rebuilding retry cache in case of HA failover"); + + "enable rebuilding retry cache in case of HA failover"), + DATANODE_ID(-48, "UUID per Datanode and distinct StorageID per storage " + + "directory."); final int lv; final int ancestorLV; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 5c2b5509606..c4dd96ccbe9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -231,11 +231,14 @@ public class PBHelper { } public static DatanodeIDProto convert(DatanodeID dn) { + // For wire compatibility with older versions we transmit the StorageID + // which is the same as the DatanodeUuid. Since StorageID is a required + // field we pass the empty string if the DatanodeUuid is not yet known. return DatanodeIDProto.newBuilder() .setIpAddr(dn.getIpAddr()) .setHostName(dn.getHostName()) - .setDatanodeUuid(dn.getDatanodeUuid()) .setXferPort(dn.getXferPort()) + .setDatanodeUuid(dn.getDatanodeUuid() != null ? dn.getDatanodeUuid() : "") .setInfoPort(dn.getInfoPort()) .setIpcPort(dn.getIpcPort()).build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 03b5681f763..a72a1b1c2c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -420,6 +420,10 @@ public class DatanodeManager { /** Get a datanode descriptor given corresponding DatanodeUUID */ DatanodeDescriptor getDatanode(final String datanodeUuid) { + if (datanodeUuid == null) { + return null; + } + return datanodeMap.get(datanodeUuid); } @@ -776,7 +780,7 @@ public class DatanodeManager { NameNode.stateChangeLog.info("BLOCK* registerDatanode: from " + nodeReg + " storage " + nodeReg.getDatanodeUuid()); - DatanodeDescriptor nodeS = datanodeMap.get(nodeReg.getDatanodeUuid()); + DatanodeDescriptor nodeS = getDatanode(nodeReg.getDatanodeUuid()); DatanodeDescriptor nodeN = host2DatanodeMap.getDatanodeByXferAddr( nodeReg.getIpAddr(), nodeReg.getXferPort()); @@ -843,13 +847,13 @@ public class DatanodeManager { } } return; - } - - // this is a new datanode serving a new data storage - if ("".equals(nodeReg.getDatanodeUuid())) { - // this data storage has never been registered - // it is either empty or was created by pre-storageID version of DFS - nodeReg.setDatanodeUuid(DatanodeStorage.newStorageID()); + } + + // This is a new datanode. + if (nodeReg.getDatanodeUuid() == null || + nodeReg.getDatanodeUuid().isEmpty()) { + // this data node has never been registered + nodeReg.generateNewDatanodeUuid(); if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug( "BLOCK* NameSystem.registerDatanode: " diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java index 2a96faaf6e0..a1882c468d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java @@ -236,6 +236,9 @@ public abstract class Storage extends StorageInfo { final boolean useLock; // flag to enable storage lock final StorageDirType dirType; // storage dir type FileLock lock; // storage lock + + //TODO HDFS-2832: Consider moving this out of StorageDirectory. + String storageUuid = null; // Storage directory identifier. public StorageDirectory(File dir) { // default dirType is null @@ -246,6 +249,14 @@ public abstract class Storage extends StorageInfo { this(dir, dirType, true); } + public void setStorageUuid(String storageUuid) { + this.storageUuid = storageUuid; + } + + public String getStorageUuid() { + return storageUuid; + } + /** * Constructor * @param dir directory corresponding to the storage diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 4d61cdfb447..62b9b7a7a80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketTimeoutException; import java.util.Collection; +import java.util.Iterator; import java.util.Map; import org.apache.commons.logging.Log; @@ -257,6 +258,7 @@ class BPServiceActor implements Runnable { /** * Report received blocks and delete hints to the Namenode + * TODO: Fix reportReceivedDeletedBlocks to send reports per-volume. * * @throws IOException */ @@ -388,23 +390,40 @@ class BPServiceActor implements Runnable { // a FINALIZED one. reportReceivedDeletedBlocks(); + // Send one block report per known storage. + // Create block report long brCreateStartTime = now(); - BlockListAsLongs bReport = dn.getFSDataset().getBlockReport( - bpos.getBlockPoolId()); + long totalBlockCount = 0; + + Map perVolumeBlockLists = + dn.getFSDataset().getBlockReports(bpos.getBlockPoolId()); // Send block report long brSendStartTime = now(); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(bpRegistration.getDatanodeUuid()), - bReport.getBlockListAsLongs()) }; - cmd = bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), report); + StorageBlockReport[] reports = + new StorageBlockReport[perVolumeBlockLists.size()]; + + int i = 0; + for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { + String storageID = kvPair.getKey(); + BlockListAsLongs blockList = kvPair.getValue(); + totalBlockCount += blockList.getNumberOfBlocks(); + + // Dummy DatanodeStorage object just for sending the block report. + DatanodeStorage dnStorage = new DatanodeStorage(storageID); + reports[i++] = + new StorageBlockReport( + dnStorage, blockList.getBlockListAsLongs()); + } + + cmd = bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), reports); // Log the block report processing stats from Datanode perspective long brSendCost = now() - brSendStartTime; long brCreateCost = brSendStartTime - brCreateStartTime; dn.getMetrics().addBlockReport(brSendCost); - LOG.info("BlockReport of " + bReport.getNumberOfBlocks() + LOG.info("BlockReport of " + totalBlockCount + " blocks took " + brCreateCost + " msec to generate and " + brSendCost + " msecs for RPC and NN processing"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index d541c6ff3e3..ca5b6817f25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -517,7 +517,7 @@ public class DataNode extends Configured directoryScanner.start(); } else { LOG.info("Periodic Directory Tree Verification scan is disabled because " + - reason); + reason); } } @@ -750,7 +750,7 @@ public class DataNode extends Configured } DatanodeID dnId = new DatanodeID( streamingAddr.getAddress().getHostAddress(), hostName, - getDatanodeUuid(), getXferPort(), getInfoPort(), getIpcPort()); + storage.getDatanodeUuid(), getXferPort(), getInfoPort(), getIpcPort()); return new DatanodeRegistration(dnId, storageInfo, new ExportedBlockKeys(), VersionInfo.getVersion()); } @@ -768,16 +768,16 @@ public class DataNode extends Configured id = bpRegistration; } - if (storage.getStorageID().equals("")) { - // This is a fresh datanode, persist the NN-provided storage ID - storage.setStorageID(bpRegistration.getDatanodeUuid()); + if (storage.getDatanodeUuid() == null) { + // This is a fresh datanode, persist the NN-provided Datanode ID + storage.setDatanodeUuid(bpRegistration.getDatanodeUuid()); storage.writeAll(); - LOG.info("New storage id " + bpRegistration.getDatanodeUuid() - + " is assigned to data-node " + bpRegistration); - } else if(!storage.getStorageID().equals(bpRegistration.getDatanodeUuid())) { - throw new IOException("Inconsistent storage IDs. Name-node returned " + LOG.info("Datanode ID " + bpRegistration.getDatanodeUuid() + + " is assigned to new storage " + bpRegistration); + } else if(!storage.getDatanodeUuid().equals(bpRegistration.getDatanodeUuid())) { + throw new IOException("Inconsistent Datanode IDs. Name-node returned " + bpRegistration.getDatanodeUuid() - + ". Expecting " + storage.getStorageID()); + + ". Expecting " + storage.getDatanodeUuid()); } registerBlockPoolWithSecretManager(bpRegistration, blockPoolId); @@ -925,10 +925,6 @@ public class DataNode extends Configured return streamingAddr.getPort(); } - String getDatanodeUuid() { - return storage.getStorageID(); - } - /** * @return name useful for logging */ @@ -1014,11 +1010,6 @@ public class DataNode extends Configured return metrics; } - public static void setNewStorageID(DatanodeID dnId) { - LOG.info("Datanode is " + dnId); - dnId.setDatanodeUuid(DatanodeStorage.newStorageID()); - } - /** Ensure the authentication method is kerberos */ private void checkKerberosAuthMethod(String msg) throws IOException { // User invoking the call must be same as the datanode user @@ -1818,7 +1809,7 @@ public class DataNode extends Configured @Override public String toString() { return "DataNode{data=" + data + ", localName='" + getDisplayName() - + "', storageID='" + getDatanodeUuid() + "', xmitsInProgress=" + + "', datanodeUuid='" + storage.getDatanodeUuid() + "', xmitsInProgress=" + xmitsInProgress.get() + "}"; } @@ -1872,7 +1863,6 @@ public class DataNode extends Configured } /** - * This method is used for testing. * Examples are adding and deleting blocks directly. * The most common usage will be when the data node's storage is simulated. * @@ -2425,6 +2415,10 @@ public class DataNode extends Configured return dnConf; } + public String getDatanodeUuid() { + return id == null ? null : id.getDatanodeUuid(); + } + boolean shouldRun() { return shouldRun; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index 7fbe13c90c2..cd84733288c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -24,13 +24,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.channels.FileLock; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Properties; +import java.util.*; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -72,8 +66,13 @@ public class DataStorage extends Storage { public final static String STORAGE_DIR_FINALIZED = "finalized"; public final static String STORAGE_DIR_TMP = "tmp"; - /** Unique storage ID. {@see DataNode#createNewStorageId(int)} for details */ - private String storageID; + /** + * Datanode UUID that this storage is currently attached to. This + * is the same as the legacy StorageID for datanodes that were + * upgraded from a pre-UUID version. For compatibility with prior + * versions of Datanodes we cannot make this field a UUID. + */ + private String datanodeUuid = null; // Flag to ensure we only initialize storage once private boolean initialized = false; @@ -85,33 +84,29 @@ public class DataStorage extends Storage { DataStorage() { super(NodeType.DATA_NODE); - storageID = ""; } public StorageInfo getBPStorage(String bpid) { return bpStorageMap.get(bpid); } - public DataStorage(StorageInfo storageInfo, String strgID) { + public DataStorage(StorageInfo storageInfo) { super(NodeType.DATA_NODE, storageInfo); - this.storageID = strgID; } - /** @return storage ID. */ - public synchronized String getStorageID() { - return storageID; + public synchronized String getDatanodeUuid() { + return datanodeUuid; } - synchronized void setStorageID(String newStorageID) { - this.storageID = newStorageID; + synchronized void setDatanodeUuid(String newDatanodeUuid) { + this.datanodeUuid = newDatanodeUuid; } /** Create an ID for this storage. */ - public synchronized void createStorageID() { - if (storageID != null && !storageID.isEmpty()) { - return; + public synchronized void createStorageID(StorageDirectory sd) { + if (sd.getStorageUuid() == null) { + sd.setStorageUuid(DatanodeStorage.newStorageID()); } - storageID = DatanodeStorage.newStorageID(); } /** @@ -164,7 +159,7 @@ public class DataStorage extends Storage { case NOT_FORMATTED: // format LOG.info("Storage directory " + dataDir + " is not formatted"); LOG.info("Formatting ..."); - format(sd, nsInfo); + format(sd, nsInfo, datanode.getDatanodeUuid()); break; default: // recovery part is common sd.doRecover(curState); @@ -193,11 +188,9 @@ public class DataStorage extends Storage { doTransition(datanode, getStorageDir(idx), nsInfo, startOpt); assert this.getLayoutVersion() == nsInfo.getLayoutVersion() : "Data-node and name-node layout versions must be the same."; + createStorageID(getStorageDir(idx)); } - // make sure we have storage id set - if not - generate new one - createStorageID(); - // 3. Update all storages. Some of them might have just been formatted. this.writeAll(); @@ -265,19 +258,28 @@ public class DataStorage extends Storage { } } - void format(StorageDirectory sd, NamespaceInfo nsInfo) throws IOException { + void format(StorageDirectory sd, NamespaceInfo nsInfo, + String datanodeUuid) throws IOException { sd.clearDirectory(); // create directory this.layoutVersion = HdfsConstants.LAYOUT_VERSION; this.clusterID = nsInfo.getClusterID(); this.namespaceID = nsInfo.getNamespaceID(); this.cTime = 0; - // store storageID as it currently is + this.datanodeUuid = datanodeUuid; + + if (sd.getStorageUuid() == null) { + // Assign a new Storage UUID. + sd.setStorageUuid(UUID.randomUUID().toString()); + } + writeProperties(sd); } /* * Set ClusterID, StorageID, StorageType, CTime into - * DataStorage VERSION file + * DataStorage VERSION file. + * Always called just before writing the properties to + * the VERSION file. */ @Override protected void setPropertiesFromFields(Properties props, @@ -287,7 +289,13 @@ public class DataStorage extends Storage { props.setProperty("clusterID", clusterID); props.setProperty("cTime", String.valueOf(cTime)); props.setProperty("layoutVersion", String.valueOf(layoutVersion)); - props.setProperty("storageID", getStorageID()); + props.setProperty("storageID", sd.getStorageUuid()); + + if (LayoutVersion.supports(Feature.DATANODE_ID, layoutVersion) && + datanodeUuid != null) { + props.setProperty("datanodeUuid", datanodeUuid); + } + // Set NamespaceID in version before federation if (!LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) { props.setProperty("namespaceID", String.valueOf(namespaceID)); @@ -297,6 +305,8 @@ public class DataStorage extends Storage { /* * Read ClusterID, StorageID, StorageType, CTime from * DataStorage VERSION file and verify them. + * Always called just after reading the properties from the VERSION + * file. */ @Override protected void setFieldsFromProperties(Properties props, StorageDirectory sd) @@ -311,20 +321,36 @@ public class DataStorage extends Storage { setNamespaceID(props, sd); } + // valid storage id, storage id may be empty String ssid = props.getProperty("storageID"); if (ssid == null) { throw new InconsistentFSStateException(sd.getRoot(), "file " + STORAGE_FILE_VERSION + " is invalid."); } - String sid = getStorageID(); - if (!(sid.equals("") || ssid.equals("") || sid.equals(ssid))) { + String sid = sd.getStorageUuid(); + if (!(sid == null || sid.equals("") || + ssid.equals("") || sid.equals(ssid))) { throw new InconsistentFSStateException(sd.getRoot(), "has incompatible storage Id."); } - - if (sid.equals("")) { // update id only if it was empty - setStorageID(ssid); + + if (sid == null) { // update id only if it was null + sd.setStorageUuid(ssid); + } + + // Update the datanode UUID if present. + if (props.getProperty("datanodeUuid") != null) { + String dnUuid = props.getProperty("datanodeUuid"); + + if (getDatanodeUuid() == null) { + setDatanodeUuid(dnUuid); + } else if (getDatanodeUuid().compareTo(dnUuid) != 0) { + throw new InconsistentFSStateException(sd.getRoot(), + "Root " + sd.getRoot() + ": DatanodeUuid=" + dnUuid + + ", does not match " + datanodeUuid + " from other" + + " StorageDirectory."); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index 4f633973111..d9f7655dd6f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@ -92,9 +92,6 @@ public interface FsDatasetSpi extends FSDatasetMBean { /** @return a volume information map (name => info). */ public Map getVolumeInfoMap(); - /** @return a list of block pools. */ - public String[] getBlockPoolList(); - /** @return a list of finalized blocks for the given block pool. */ public List getFinalizedBlocks(String bpid); @@ -262,6 +259,7 @@ public interface FsDatasetSpi extends FSDatasetMBean { public void unfinalizeBlock(ExtendedBlock b) throws IOException; /** + * TODO HDFS-2832: Deprecate this when we fix tests. * Returns the block report - the full list of blocks stored under a * block pool * @param bpid Block Pool Id @@ -269,6 +267,13 @@ public interface FsDatasetSpi extends FSDatasetMBean { */ public BlockListAsLongs getBlockReport(String bpid); + /** + * Returns one block report per volume. + * @param bpid Block Pool Id + * @return - a map of StorageID to block report for the volume. + */ + public Map getBlockReports(String bpid); + /** Does the dataset contain the block? */ public boolean contains(ExtendedBlock block); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 5218dd75db1..2d84e6ac843 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -43,7 +43,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; @@ -53,6 +52,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; +import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader; import org.apache.hadoop.hdfs.server.datanode.DataBlockScanner; import org.apache.hadoop.hdfs.server.datanode.DataNode; @@ -170,10 +170,16 @@ class FsDatasetImpl implements FsDatasetSpi { final DataNode datanode; final FsVolumeList volumes; - final ReplicaMap volumeMap; final FsDatasetAsyncDiskService asyncDiskService; private final int validVolsRequired; + // TODO HDFS-2832: Consider removing duplicated block info from these + // two maps. This might require some refactoring + // rewrite of FsDatasetImpl. + final ReplicaMap volumeMap; + final Map perVolumeReplicaMap; + + // Used for synchronizing access to usage stats private final Object statsLock = new Object(); @@ -211,16 +217,18 @@ class FsDatasetImpl implements FsDatasetSpi { final List volArray = new ArrayList( storage.getNumStorageDirs()); for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) { - final File dir = storage.getStorageDir(idx).getCurrentDir(); // TODO: getStorageTypeFromLocations() is only a temporary workaround and // should be replaced with getting storage type from DataStorage (missing // storage type now) directly. + Storage.StorageDirectory sd = storage.getStorageDir(idx); + final File dir = sd.getCurrentDir(); final StorageType storageType = getStorageTypeFromLocations(dataLocations, dir); - volArray.add(new FsVolumeImpl(this, storage.getStorageID(), dir, conf, + volArray.add(new FsVolumeImpl(this, sd.getStorageUuid(), dir, conf, storageType)); LOG.info("Added volume - " + dir + ", StorageType: " + storageType); } volumeMap = new ReplicaMap(this); + perVolumeReplicaMap = new HashMap(); @SuppressWarnings("unchecked") final VolumeChoosingPolicy blockChooserImpl = @@ -229,14 +237,14 @@ class FsDatasetImpl implements FsDatasetSpi { RoundRobinVolumeChoosingPolicy.class, VolumeChoosingPolicy.class), conf); volumes = new FsVolumeList(volArray, volsFailed, blockChooserImpl); - volumes.getVolumeMap(volumeMap); + volumes.initializeReplicaMaps(perVolumeReplicaMap, volumeMap, this); File[] roots = new File[storage.getNumStorageDirs()]; for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) { roots[idx] = storage.getStorageDir(idx).getCurrentDir(); } asyncDiskService = new FsDatasetAsyncDiskService(datanode, roots); - registerMBean(storage.getStorageID()); + registerMBean(datanode.getDatanodeUuid()); } private StorageType getStorageTypeFromLocations( @@ -326,9 +334,6 @@ class FsDatasetImpl implements FsDatasetSpi { File getBlockFile(String bpid, Block b) throws IOException { File f = validateBlockFile(bpid, b); if(f == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("b=" + b + ", volumeMap=" + volumeMap); - } throw new IOException("Block " + b + " is not valid."); } return f; @@ -602,6 +607,7 @@ class FsDatasetImpl implements FsDatasetSpi { // Replace finalized replica by a RBW replica in replicas map volumeMap.add(bpid, newReplicaInfo); + perVolumeReplicaMap.get(v).add(bpid, newReplicaInfo); return newReplicaInfo; } @@ -731,6 +737,7 @@ class FsDatasetImpl implements FsDatasetSpi { ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(b.getBlockId(), b.getGenerationStamp(), v, f.getParentFile()); volumeMap.add(b.getBlockPoolId(), newReplicaInfo); + perVolumeReplicaMap.get(v).add(b.getBlockPoolId(), newReplicaInfo); return newReplicaInfo; } @@ -849,6 +856,7 @@ class FsDatasetImpl implements FsDatasetSpi { rbw.setBytesAcked(visible); // overwrite the RBW in the volume map volumeMap.add(b.getBlockPoolId(), rbw); + perVolumeReplicaMap.get(v).add(b.getBlockPoolId(), rbw); return rbw; } @@ -868,6 +876,7 @@ class FsDatasetImpl implements FsDatasetSpi { ReplicaInPipeline newReplicaInfo = new ReplicaInPipeline(b.getBlockId(), b.getGenerationStamp(), v, f.getParentFile()); volumeMap.add(b.getBlockPoolId(), newReplicaInfo); + perVolumeReplicaMap.get(v).add(b.getBlockPoolId(), newReplicaInfo); return newReplicaInfo; } @@ -936,6 +945,7 @@ class FsDatasetImpl implements FsDatasetSpi { newReplicaInfo = new FinalizedReplica(replicaInfo, v, dest.getParentFile()); } volumeMap.add(bpid, newReplicaInfo); + perVolumeReplicaMap.get(newReplicaInfo.getVolume()).add(bpid, newReplicaInfo); return newReplicaInfo; } @@ -949,6 +959,8 @@ class FsDatasetImpl implements FsDatasetSpi { if (replicaInfo != null && replicaInfo.getState() == ReplicaState.TEMPORARY) { // remove from volumeMap volumeMap.remove(b.getBlockPoolId(), b.getLocalBlock()); + perVolumeReplicaMap.get((FsVolumeImpl) replicaInfo.getVolume()) + .remove(b.getBlockPoolId(), b.getLocalBlock()); // delete the on-disk temp file if (delBlockFromDisk(replicaInfo.getBlockFile(), @@ -983,12 +995,9 @@ class FsDatasetImpl implements FsDatasetSpi { return true; } - /** - * Generates a block report from the in-memory block map. - */ - @Override // FsDatasetSpi - public BlockListAsLongs getBlockReport(String bpid) { - int size = volumeMap.size(bpid); + private BlockListAsLongs getBlockReportWithReplicaMap( + String bpid, ReplicaMap rMap) { + int size = rMap.size(bpid); ArrayList finalized = new ArrayList(size); ArrayList uc = new ArrayList(); if (size == 0) { @@ -996,7 +1005,7 @@ class FsDatasetImpl implements FsDatasetSpi { } synchronized(this) { - for (ReplicaInfo b : volumeMap.replicas(bpid)) { + for (ReplicaInfo b : rMap.replicas(bpid)) { switch(b.getState()) { case FINALIZED: finalized.add(b); @@ -1019,6 +1028,28 @@ class FsDatasetImpl implements FsDatasetSpi { } } + /** + * Generates a block report from the in-memory block map. + */ + @Override // FsDatasetSpi + public BlockListAsLongs getBlockReport(String bpid) { + return getBlockReportWithReplicaMap(bpid, volumeMap); + } + + @Override + public Map getBlockReports(String bpid) { + Map blockReportMap = + new HashMap(); + + for (FsVolumeImpl v : getVolumes()) { + ReplicaMap rMap = perVolumeReplicaMap.get(v); + BlockListAsLongs blockList = getBlockReportWithReplicaMap(bpid, rMap); + blockReportMap.put(v.getStorageID(), blockList); + } + + return blockReportMap; + } + /** * Get the list of finalized blocks from in-memory blockmap for a block pool. */ @@ -1159,6 +1190,7 @@ class FsDatasetImpl implements FsDatasetSpi { v.clearPath(bpid, parent); } volumeMap.remove(bpid, invalidBlks[i]); + perVolumeReplicaMap.get(v).remove(bpid, invalidBlks[i]); } // Delete the block asynchronously to make sure we can do it fast enough @@ -1220,6 +1252,7 @@ class FsDatasetImpl implements FsDatasetSpi { LOG.warn("Removing replica " + bpid + ":" + b.getBlockId() + " on failed volume " + fv.getCurrentDir().getAbsolutePath()); ib.remove(); + perVolumeReplicaMap.get(fv).remove(bpid, b.getBlockId()); removedBlocks++; } } @@ -1248,22 +1281,15 @@ class FsDatasetImpl implements FsDatasetSpi { /** * Register the FSDataset MBean using the name - * "hadoop:service=DataNode,name=FSDatasetState-" + * "hadoop:service=DataNode,name=FSDatasetState-" */ - void registerMBean(final String storageId) { + void registerMBean(final String datanodeUuid) { // We wrap to bypass standard mbean naming convetion. // This wraping can be removed in java 6 as it is more flexible in // package naming for mbeans and their impl. - StandardMBean bean; - String storageName; - if (storageId == null || storageId.equals("")) {// Temp fix for the uninitialized storage - storageName = "UndefinedStorageId" + DFSUtil.getRandom().nextInt(); - } else { - storageName = storageId; - } try { - bean = new StandardMBean(this,FSDatasetMBean.class); - mbeanName = MBeans.register("DataNode", "FSDatasetState-" + storageName, bean); + StandardMBean bean = new StandardMBean(this,FSDatasetMBean.class); + mbeanName = MBeans.register("DataNode", "FSDatasetState-" + datanodeUuid, bean); } catch (NotCompliantMBeanException e) { LOG.warn("Error registering FSDatasetState MBean", e); } @@ -1343,6 +1369,8 @@ class FsDatasetImpl implements FsDatasetSpi { // Block is in memory and not on the disk // Remove the block from volumeMap volumeMap.remove(bpid, blockId); + perVolumeReplicaMap.get((FsVolumeImpl) memBlockInfo.getVolume()) + .remove(bpid, blockId); final DataBlockScanner blockScanner = datanode.getBlockScanner(); if (blockScanner != null) { blockScanner.deleteBlock(bpid, new Block(blockId)); @@ -1366,6 +1394,8 @@ class FsDatasetImpl implements FsDatasetSpi { ReplicaInfo diskBlockInfo = new FinalizedReplica(blockId, diskFile.length(), diskGS, vol, diskFile.getParentFile()); volumeMap.add(bpid, diskBlockInfo); + perVolumeReplicaMap.get((FsVolumeImpl) memBlockInfo.getVolume()). + remove(bpid, diskBlockInfo); final DataBlockScanner blockScanner = datanode.getBlockScanner(); if (blockScanner != null) { blockScanner.addBlock(new ExtendedBlock(bpid, diskBlockInfo)); @@ -1639,7 +1669,14 @@ class FsDatasetImpl implements FsDatasetSpi { LOG.info("Adding block pool " + bpid); volumes.addBlockPool(bpid, conf); volumeMap.initBlockPool(bpid); - volumes.getVolumeMap(bpid, volumeMap); + volumes.getAllVolumesMap(bpid, volumeMap); + + // TODO: Avoid the double scan. + for (FsVolumeImpl v : getVolumes()) { + ReplicaMap rMap = perVolumeReplicaMap.get(v); + rMap.initBlockPool(bpid); + volumes.getVolumeMap(bpid, v, rMap); + } } @Override @@ -1649,11 +1686,6 @@ class FsDatasetImpl implements FsDatasetSpi { volumes.removeBlockPool(bpid); } - @Override - public String[] getBlockPoolList() { - return volumeMap.getBlockPoolList(); - } - /** * Class for representing the Datanode volume information */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index 05dc7bdfef8..30574b3b6d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -290,7 +290,7 @@ class FsVolumeImpl implements FsVolumeSpi { } } - String getStorageID() { + public String getStorageID() { return storageID; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index 90d2aee8e48..a0e65db3647 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -18,10 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; +import java.util.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; @@ -93,27 +90,37 @@ class FsVolumeList { return remaining; } - void getVolumeMap(ReplicaMap volumeMap) throws IOException { + void initializeReplicaMaps(Map perVolumeReplicaMap, + ReplicaMap globalReplicaMap, + Object mutex) throws IOException { for (FsVolumeImpl v : volumes) { - v.getVolumeMap(volumeMap); + ReplicaMap rMap = new ReplicaMap(mutex); + v.getVolumeMap(rMap); + perVolumeReplicaMap.put(v, rMap); + globalReplicaMap.addAll(rMap); } } - void getVolumeMap(String bpid, ReplicaMap volumeMap) throws IOException { + void getAllVolumesMap(String bpid, ReplicaMap volumeMap) throws IOException { long totalStartTime = System.currentTimeMillis(); for (FsVolumeImpl v : volumes) { - FsDatasetImpl.LOG.info("Adding replicas to map for block pool " + bpid + - " on volume " + v + "..."); - long startTime = System.currentTimeMillis(); - v.getVolumeMap(bpid, volumeMap); - long timeTaken = System.currentTimeMillis() - startTime; - FsDatasetImpl.LOG.info("Time to add replicas to map for block pool " + bpid + - " on volume " + v + ": " + timeTaken + "ms"); + getVolumeMap(bpid, v, volumeMap); } long totalTimeTaken = System.currentTimeMillis() - totalStartTime; FsDatasetImpl.LOG.info("Total time to add all replicas to map: " + totalTimeTaken + "ms"); } + + void getVolumeMap(String bpid, FsVolumeImpl volume, ReplicaMap volumeMap) + throws IOException { + FsDatasetImpl.LOG.info("Adding replicas to map for block pool " + bpid + + " on volume " + volume + "..."); + long startTime = System.currentTimeMillis(); + volume.getVolumeMap(bpid, volumeMap); + long timeTaken = System.currentTimeMillis() - startTime; + FsDatasetImpl.LOG.info("Time to add replicas to map for block pool " + bpid + + " on volume " + volume + ": " + timeTaken + "ms"); + } /** * Calls {@link FsVolumeImpl#checkDirs()} on each volume, removing any diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java index aa81c727071..d59c38644ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java @@ -117,6 +117,15 @@ class ReplicaMap { return m.put(replicaInfo.getBlockId(), replicaInfo); } } + + /** + * Add all entries from the given replica map into the local replica + * map. + * @param + */ + void addAll(ReplicaMap other) { + map.putAll(other.map); + } /** * Remove the replica's meta information from the map that matches diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 268d344f5ea..df28513b23f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -289,6 +289,7 @@ public class JsonUtil { return null; } + // TODO: Fix storageID final Map m = new TreeMap(); m.put("ipAddr", datanodeinfo.getIpAddr()); m.put("hostName", datanodeinfo.getHostName()); @@ -314,6 +315,7 @@ public class JsonUtil { return null; } + // TODO: Fix storageID return new DatanodeInfo( (String)m.get("ipAddr"), (String)m.get("hostName"), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 123ebec77ea..0c243ade507 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -41,13 +41,7 @@ import java.net.Socket; import java.net.URL; import java.net.URLConnection; import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; +import java.util.*; import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; @@ -805,7 +799,8 @@ public class DFSTestUtil { } private static DatanodeID getDatanodeID(String ipAddr) { - return new DatanodeID(ipAddr, "localhost", "", + return new DatanodeID(ipAddr, "localhost", + UUID.randomUUID().toString(), DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT); @@ -816,8 +811,8 @@ public class DFSTestUtil { } public static DatanodeID getLocalDatanodeID(int port) { - return new DatanodeID("127.0.0.1", "localhost", "", - port, port, port); + return new DatanodeID("127.0.0.1", "localhost", + UUID.randomUUID().toString(), port, port, port); } public static DatanodeDescriptor getLocalDatanodeDescriptor() { @@ -838,8 +833,9 @@ public class DFSTestUtil { public static DatanodeInfo getDatanodeInfo(String ipAddr, String host, int port) { - return new DatanodeInfo(new DatanodeID(ipAddr, host, "", - port, DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, + return new DatanodeInfo(new DatanodeID(ipAddr, host, + UUID.randomUUID().toString(), port, + DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT)); } @@ -893,7 +889,8 @@ public class DFSTestUtil { public static DatanodeDescriptor getDatanodeDescriptor(String ipAddr, int port, String rackLocation) { - DatanodeID dnId = new DatanodeID(ipAddr, "host", "", port, + DatanodeID dnId = new DatanodeID(ipAddr, "host", + UUID.randomUUID().toString(), port, DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT); return new DatanodeDescriptor(dnId, rackLocation); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java index e465ed3b8e3..767005fd82b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeRegistration.java @@ -172,7 +172,7 @@ public class TestDatanodeRegistration { // register a datanode DatanodeID dnId = new DatanodeID(DN_IP_ADDR, DN_HOSTNAME, - "fake-storage-id", DN_XFER_PORT, DN_INFO_PORT, DN_IPC_PORT); + "fake-datanode-id", DN_XFER_PORT, DN_INFO_PORT, DN_IPC_PORT); long nnCTime = cluster.getNamesystem().getFSImage().getStorage() .getCTime(); StorageInfo mockStorageInfo = mock(StorageInfo.class); @@ -188,7 +188,7 @@ public class TestDatanodeRegistration { // register the same datanode again with a different storage ID dnId = new DatanodeID(DN_IP_ADDR, DN_HOSTNAME, - "changed-fake-storage-id", DN_XFER_PORT, DN_INFO_PORT, DN_IPC_PORT); + "changed-fake-datanode-id", DN_XFER_PORT, DN_INFO_PORT, DN_IPC_PORT); dnReg = new DatanodeRegistration(dnId, mockStorageInfo, null, VersionInfo.getVersion()); rpcServer.registerDatanode(dnReg); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java index 7836bc66805..fa5e103bec5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPeerCache.java @@ -149,7 +149,7 @@ public class TestPeerCache { public void testAddAndRetrieve() throws Exception { PeerCache cache = new PeerCache(3, 100000); DatanodeID dnId = new DatanodeID("192.168.0.1", - "fakehostname", "fake_storage_id", + "fakehostname", "fake_datanode_id", 100, 101, 102); FakePeer peer = new FakePeer(dnId, false); cache.put(dnId, peer); @@ -169,7 +169,7 @@ public class TestPeerCache { FakePeer peers[] = new FakePeer[CAPACITY]; for (int i = 0; i < CAPACITY; ++i) { dnIds[i] = new DatanodeID("192.168.0.1", - "fakehostname_" + i, "fake_storage_id", + "fakehostname_" + i, "fake_datanode_id", 100, 101, 102); peers[i] = new FakePeer(dnIds[i], false); } @@ -200,7 +200,7 @@ public class TestPeerCache { FakePeer peers[] = new FakePeer[CAPACITY + 1]; for (int i = 0; i < dnIds.length; ++i) { dnIds[i] = new DatanodeID("192.168.0.1", - "fakehostname_" + i, "fake_storage_id_" + i, + "fakehostname_" + i, "fake_datanode_id_" + i, 100, 101, 102); peers[i] = new FakePeer(dnIds[i], false); } @@ -231,7 +231,7 @@ public class TestPeerCache { final int CAPACITY = 3; PeerCache cache = new PeerCache(CAPACITY, 100000); DatanodeID dnId = new DatanodeID("192.168.0.1", - "fakehostname", "fake_storage_id", + "fakehostname", "fake_datanode_id", 100, 101, 102); HashMultiset peers = HashMultiset.create(CAPACITY); for (int i = 0; i < CAPACITY; ++i) { @@ -256,7 +256,7 @@ public class TestPeerCache { final int CAPACITY = 3; PeerCache cache = new PeerCache(CAPACITY, 100000); DatanodeID dnId = new DatanodeID("192.168.0.1", - "fakehostname", "fake_storage_id", + "fakehostname", "fake_datanode_id", 100, 101, 102); HashMultiset peers = HashMultiset.create(CAPACITY); for (int i = 0; i < CAPACITY; ++i) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java index 1a676897a16..be111659255 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java @@ -453,7 +453,7 @@ public class UpgradeUtilities { */ public static void createDataNodeVersionFile(File[] parent, StorageInfo version, String bpid, String bpidToWrite) throws IOException { - DataStorage storage = new DataStorage(version, "doNotCare"); + DataStorage storage = new DataStorage(version); File[] versionFiles = new File[parent.length]; for (int i = 0; i < parent.length; i++) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index e0d5698f76f..6878eecc3b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -527,6 +527,8 @@ public class TestBlockManager { public void testSafeModeIBR() throws Exception { DatanodeDescriptor node = spy(nodes.get(0)); DatanodeStorageInfo ds = node.getStorageInfos()[0]; + + // TODO: Needs to be fixed. DatanodeUuid is not storageID. node.setDatanodeUuid(ds.getStorageID()); node.isAlive = true; @@ -571,7 +573,10 @@ public class TestBlockManager { public void testSafeModeIBRAfterIncremental() throws Exception { DatanodeDescriptor node = spy(nodes.get(0)); DatanodeStorageInfo ds = node.getStorageInfos()[0]; + + // TODO: Needs to be fixed. DatanodeUuid is not storageID. node.setDatanodeUuid(ds.getStorageID()); + node.isAlive = true; DatanodeRegistration nodeReg = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java index ab6ed12492e..0bf79594c9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java @@ -445,9 +445,9 @@ public class TestJspHelper { @Test public void testSortNodeByFields() throws Exception { - DatanodeID dnId1 = new DatanodeID("127.0.0.1", "localhost1", "storage1", + DatanodeID dnId1 = new DatanodeID("127.0.0.1", "localhost1", "datanode1", 1234, 2345, 3456); - DatanodeID dnId2 = new DatanodeID("127.0.0.2", "localhost2", "storage2", + DatanodeID dnId2 = new DatanodeID("127.0.0.2", "localhost2", "datanode2", 1235, 2346, 3457); DatanodeDescriptor dnDesc1 = new DatanodeDescriptor(dnId1, "rack1", 1024, 100, 924, 100, 10, 2); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 1f24375a0ea..c0a9481c249 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -383,8 +383,10 @@ public class SimulatedFSDataset implements FsDatasetSpi { public SimulatedFSDataset(DataStorage storage, Configuration conf) { if (storage != null) { - storage.createStorageID(); - this.datanodeUuid = storage.getStorageID(); + for (int i = 0; i < storage.getNumStorageDirs(); ++i) { + storage.createStorageID(storage.getStorageDir(i)); + } + this.datanodeUuid = storage.getDatanodeUuid(); } else { this.datanodeUuid = "unknownStorageId-" + UUID.randomUUID(); } @@ -459,6 +461,15 @@ public class SimulatedFSDataset implements FsDatasetSpi { return new BlockListAsLongs(blocks, null); } + @Override + public synchronized Map getBlockReports( + String bpid) { + Map reports = + new HashMap(); + reports.put("", getBlockReport(bpid)); + return reports; + } + @Override // FSDatasetMBean public long getCapacity() { return storage.getCapacity(); @@ -910,6 +921,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { public String updateReplicaUnderRecovery(ExtendedBlock oldBlock, long recoveryId, long newlength) { + // Caller does not care about the exact Storage UUID returned. return datanodeUuid; } @@ -964,11 +976,6 @@ public class SimulatedFSDataset implements FsDatasetSpi { throw new UnsupportedOperationException(); } - @Override - public String[] getBlockPoolList() { - throw new UnsupportedOperationException(); - } - @Override public void checkAndUpdate(String bpid, long blockId, File diskFile, File diskMetaFile, FsVolumeSpi vol) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java index 1bc8db1e253..0e60670a412 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java @@ -368,7 +368,7 @@ public class TestBlockReport { int randIndex = rand.nextInt(blocks.size()); // Get a block and screw its GS Block corruptedBlock = blocks.get(randIndex); - String secondNode = cluster.getDataNodes().get(DN_N1).getDatanodeUuid(); + String secondNode = cluster.getDataNodes().get(DN_N1).getDatanodeId().getDatanodeUuid(); if(LOG.isDebugEnabled()) { LOG.debug("Working with " + secondNode); LOG.debug("BlockGS before " + blocks.get(randIndex).getGenerationStamp()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java index f88015c2954..9de5ff3e63a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java @@ -840,9 +840,9 @@ public class NNThroughputBenchmark implements Tool { "", getNodePort(dnIdx), DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT), - new DataStorage(nsInfo, ""), + new DataStorage(nsInfo), new ExportedBlockKeys(), VersionInfo.getVersion()); - DataNode.setNewStorageID(dnRegistration); + // TODO: Fix NNThroughputBenchmark. // register datanode dnRegistration = nameNodeProto.registerDatanode(dnRegistration); //first block reports @@ -939,7 +939,7 @@ public class NNThroughputBenchmark implements Tool { DatanodeInfo dnInfo = blockTargets[t]; DatanodeRegistration receivedDNReg; receivedDNReg = new DatanodeRegistration(dnInfo, - new DataStorage(nsInfo, dnInfo.getDatanodeUuid()), + new DataStorage(nsInfo), new ExportedBlockKeys(), VersionInfo.getVersion()); ReceivedDeletedBlockInfo[] rdBlocks = { new ReceivedDeletedBlockInfo( From 12ed24af66538e852026a78602a6a7962cfec830 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Mon, 21 Oct 2013 23:48:08 +0000 Subject: [PATCH 17/76] HDFS-4885. Improve the verifyBlockPlacement() API in BlockPlacementPolicy. Contributed by Junping Du git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1534417 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 + .../blockmanagement/BlockPlacementPolicy.java | 15 ++-- .../BlockPlacementPolicyDefault.java | 12 +-- .../blockmanagement/BlockPlacementStatus.java | 42 +++++++++++ .../BlockPlacementStatusDefault.java | 44 +++++++++++ .../hdfs/server/namenode/NamenodeFsck.java | 12 +-- .../hadoop/hdfs/server/namenode/TestFsck.java | 75 ++++++++++++++++++- 7 files changed, 182 insertions(+), 21 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatus.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 3b975f242b5..f62615f85d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -36,3 +36,6 @@ IMPROVEMENTS: HDFS-4988. Datanode must support all the volumes as individual storages. (Arpit Agarwal) + + HDFS-4885. Improve the verifyBlockPlacement() API in BlockPlacementPolicy. + (Junping Du via szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java index 5d9262d6022..73f474cba4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java @@ -100,18 +100,17 @@ public abstract class BlockPlacementPolicy { } /** - * Verify that the block is replicated on at least minRacks different racks - * if there is more than minRacks rack in the system. + * Verify if the block's placement meets requirement of placement policy, + * i.e. replicas are placed on no less than minRacks racks in the system. * * @param srcPath the full pathname of the file to be verified * @param lBlk block with locations - * @param minRacks number of racks the block should be replicated to - * @return the difference between the required and the actual number of racks - * the block is replicated to. + * @param numOfReplicas replica number of file to be verified + * @return the result of verification */ - abstract public int verifyBlockPlacement(String srcPath, - LocatedBlock lBlk, - int minRacks); + abstract public BlockPlacementStatus verifyBlockPlacement(String srcPath, + LocatedBlock lBlk, + int numOfReplicas); /** * Decide whether deleting the specified replica of the block still makes * the block conform to the configured block placement policy. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 493e6f87c26..99a40e38b3b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -698,22 +698,22 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { } @Override - public int verifyBlockPlacement(String srcPath, - LocatedBlock lBlk, - int minRacks) { + public BlockPlacementStatus verifyBlockPlacement(String srcPath, + LocatedBlock lBlk, int numberOfReplicas) { DatanodeInfo[] locs = lBlk.getLocations(); if (locs == null) locs = DatanodeDescriptor.EMPTY_ARRAY; int numRacks = clusterMap.getNumOfRacks(); if(numRacks <= 1) // only one rack - return 0; - minRacks = Math.min(minRacks, numRacks); + return new BlockPlacementStatusDefault( + Math.min(numRacks, numberOfReplicas), numRacks); + int minRacks = Math.min(2, numberOfReplicas); // 1. Check that all locations are different. // 2. Count locations on different racks. Set racks = new TreeSet(); for (DatanodeInfo dn : locs) racks.add(dn.getNetworkLocation()); - return minRacks - racks.size(); + return new BlockPlacementStatusDefault(racks.size(), minRacks); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatus.java new file mode 100644 index 00000000000..e2ac54a3537 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatus.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.blockmanagement; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface BlockPlacementStatus { + + /** + * Boolean value to identify if replicas of this block satisfy requirement of + * placement policy + * @return if replicas satisfy placement policy's requirement + */ + public boolean isPlacementPolicySatisfied(); + + /** + * Get description info for log or printed in case replicas are failed to meet + * requirement of placement policy + * @return description in case replicas are failed to meet requirement of + * placement policy + */ + public String getErrorDescription(); + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java new file mode 100644 index 00000000000..0b8b9659601 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.blockmanagement; + +public class BlockPlacementStatusDefault implements BlockPlacementStatus { + + private int requiredRacks = 0; + private int currentRacks = 0; + + public BlockPlacementStatusDefault(int currentRacks, int requiredRacks){ + this.requiredRacks = requiredRacks; + this.currentRacks = currentRacks; + } + + @Override + public boolean isPlacementPolicySatisfied() { + return requiredRacks <= currentRacks; + } + + @Override + public String getErrorDescription() { + if (isPlacementPolicySatisfied()) { + return null; + } + return "Block should be additionally replicated on " + + (requiredRacks - currentRacks) + " more rack(s)."; + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index 14582dccd09..b933387a31b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.net.NetUtils; @@ -374,9 +375,10 @@ public class NamenodeFsck { locs.length + " replica(s)."); } // verify block placement policy - int missingRacks = BlockPlacementPolicy.getInstance(conf, null, networktopology). - verifyBlockPlacement(path, lBlk, Math.min(2,targetFileReplication)); - if (missingRacks > 0) { + BlockPlacementStatus blockPlacementStatus = + BlockPlacementPolicy.getInstance(conf, null, networktopology). + verifyBlockPlacement(path, lBlk, targetFileReplication); + if (!blockPlacementStatus.isPlacementPolicySatisfied()) { res.numMisReplicatedBlocks++; misReplicatedPerFile++; if (!showFiles) { @@ -385,9 +387,7 @@ public class NamenodeFsck { out.print(path + ": "); } out.println(" Replica placement policy is violated for " + - block + - ". Block should be additionally replicated on " + - missingRacks + " more rack(s)."); + block + ". " + blockPlacementStatus.getErrorDescription()); } report.append(i + ". " + blkName + " len=" + block.getNumBytes()); if (locs.length == 0) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 9aaeb74a1c3..bcebce4e201 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -83,7 +83,6 @@ import org.apache.log4j.RollingFileAppender; import org.junit.Test; import com.google.common.collect.Sets; -import org.mockito.Mockito; import static org.mockito.Mockito.*; /** @@ -892,6 +891,80 @@ public class TestFsck { } } } + + /** + * Tests that the # of misreplaced replicas is correct + * @throws IOException + */ + @Test + public void testFsckMisPlacedReplicas() throws IOException { + // Desired replication factor + final short REPL_FACTOR = 2; + // Number of replicas to actually start + short NUM_DN = 2; + // Number of blocks to write + final short NUM_BLOCKS = 3; + // Set a small-ish blocksize + final long blockSize = 512; + + String [] racks = {"/rack1", "/rack1"}; + String [] hosts = {"host1", "host2"}; + + Configuration conf = new Configuration(); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + + MiniDFSCluster cluster = null; + DistributedFileSystem dfs = null; + + try { + // Startup a minicluster + cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts) + .racks(racks).build(); + assertNotNull("Failed Cluster Creation", cluster); + cluster.waitClusterUp(); + dfs = (DistributedFileSystem) cluster.getFileSystem(); + assertNotNull("Failed to get FileSystem", dfs); + + // Create a file that will be intentionally under-replicated + final String pathString = new String("/testfile"); + final Path path = new Path(pathString); + long fileLen = blockSize * NUM_BLOCKS; + DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); + + // Create an under-replicated file + NameNode namenode = cluster.getNameNode(); + NetworkTopology nettop = cluster.getNamesystem().getBlockManager() + .getDatanodeManager().getNetworkTopology(); + // Add a new node on different rack, so previous blocks' replicas + // are considered to be misplaced + nettop.add(DFSTestUtil.getDatanodeDescriptor("/rack2", "/host3")); + NUM_DN++; + + Map pmap = new HashMap(); + Writer result = new StringWriter(); + PrintWriter out = new PrintWriter(result, true); + InetAddress remoteAddress = InetAddress.getLocalHost(); + NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, + NUM_DN, (short)REPL_FACTOR, remoteAddress); + + // Run the fsck and check the Result + final HdfsFileStatus file = + namenode.getRpcServer().getFileInfo(pathString); + assertNotNull(file); + Result res = new Result(conf); + fsck.check(pathString, file, res); + // check misReplicatedBlock number. + assertEquals(res.numMisReplicatedBlocks, NUM_BLOCKS); + } finally { + if(dfs != null) { + dfs.close(); + } + if(cluster != null) { + cluster.shutdown(); + } + } + } /** Test fsck with FileNotFound */ @Test From 40fec97bef8c5a111d2eff9e9cbd6ac01905e66d Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 22 Oct 2013 00:19:04 +0000 Subject: [PATCH 18/76] svn merge -c -1534417 . for reverting HDFS-4885, which should be committed to trunk. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1534424 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 - .../blockmanagement/BlockPlacementPolicy.java | 15 ++-- .../BlockPlacementPolicyDefault.java | 12 +-- .../blockmanagement/BlockPlacementStatus.java | 42 ----------- .../BlockPlacementStatusDefault.java | 44 ----------- .../hdfs/server/namenode/NamenodeFsck.java | 12 +-- .../hadoop/hdfs/server/namenode/TestFsck.java | 75 +------------------ 7 files changed, 21 insertions(+), 182 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatus.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index f62615f85d0..3b975f242b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -36,6 +36,3 @@ IMPROVEMENTS: HDFS-4988. Datanode must support all the volumes as individual storages. (Arpit Agarwal) - - HDFS-4885. Improve the verifyBlockPlacement() API in BlockPlacementPolicy. - (Junping Du via szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java index 73f474cba4e..5d9262d6022 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java @@ -100,17 +100,18 @@ public abstract class BlockPlacementPolicy { } /** - * Verify if the block's placement meets requirement of placement policy, - * i.e. replicas are placed on no less than minRacks racks in the system. + * Verify that the block is replicated on at least minRacks different racks + * if there is more than minRacks rack in the system. * * @param srcPath the full pathname of the file to be verified * @param lBlk block with locations - * @param numOfReplicas replica number of file to be verified - * @return the result of verification + * @param minRacks number of racks the block should be replicated to + * @return the difference between the required and the actual number of racks + * the block is replicated to. */ - abstract public BlockPlacementStatus verifyBlockPlacement(String srcPath, - LocatedBlock lBlk, - int numOfReplicas); + abstract public int verifyBlockPlacement(String srcPath, + LocatedBlock lBlk, + int minRacks); /** * Decide whether deleting the specified replica of the block still makes * the block conform to the configured block placement policy. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 99a40e38b3b..493e6f87c26 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -698,22 +698,22 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { } @Override - public BlockPlacementStatus verifyBlockPlacement(String srcPath, - LocatedBlock lBlk, int numberOfReplicas) { + public int verifyBlockPlacement(String srcPath, + LocatedBlock lBlk, + int minRacks) { DatanodeInfo[] locs = lBlk.getLocations(); if (locs == null) locs = DatanodeDescriptor.EMPTY_ARRAY; int numRacks = clusterMap.getNumOfRacks(); if(numRacks <= 1) // only one rack - return new BlockPlacementStatusDefault( - Math.min(numRacks, numberOfReplicas), numRacks); - int minRacks = Math.min(2, numberOfReplicas); + return 0; + minRacks = Math.min(minRacks, numRacks); // 1. Check that all locations are different. // 2. Count locations on different racks. Set racks = new TreeSet(); for (DatanodeInfo dn : locs) racks.add(dn.getNetworkLocation()); - return new BlockPlacementStatusDefault(racks.size(), minRacks); + return minRacks - racks.size(); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatus.java deleted file mode 100644 index e2ac54a3537..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatus.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.blockmanagement; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -@InterfaceAudience.Private -@InterfaceStability.Evolving -public interface BlockPlacementStatus { - - /** - * Boolean value to identify if replicas of this block satisfy requirement of - * placement policy - * @return if replicas satisfy placement policy's requirement - */ - public boolean isPlacementPolicySatisfied(); - - /** - * Get description info for log or printed in case replicas are failed to meet - * requirement of placement policy - * @return description in case replicas are failed to meet requirement of - * placement policy - */ - public String getErrorDescription(); - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java deleted file mode 100644 index 0b8b9659601..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementStatusDefault.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.blockmanagement; - -public class BlockPlacementStatusDefault implements BlockPlacementStatus { - - private int requiredRacks = 0; - private int currentRacks = 0; - - public BlockPlacementStatusDefault(int currentRacks, int requiredRacks){ - this.requiredRacks = requiredRacks; - this.currentRacks = currentRacks; - } - - @Override - public boolean isPlacementPolicySatisfied() { - return requiredRacks <= currentRacks; - } - - @Override - public String getErrorDescription() { - if (isPlacementPolicySatisfied()) { - return null; - } - return "Block should be additionally replicated on " + - (requiredRacks - currentRacks) + " more rack(s)."; - } - -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index b933387a31b..14582dccd09 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -50,7 +50,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.net.NetUtils; @@ -375,10 +374,9 @@ public class NamenodeFsck { locs.length + " replica(s)."); } // verify block placement policy - BlockPlacementStatus blockPlacementStatus = - BlockPlacementPolicy.getInstance(conf, null, networktopology). - verifyBlockPlacement(path, lBlk, targetFileReplication); - if (!blockPlacementStatus.isPlacementPolicySatisfied()) { + int missingRacks = BlockPlacementPolicy.getInstance(conf, null, networktopology). + verifyBlockPlacement(path, lBlk, Math.min(2,targetFileReplication)); + if (missingRacks > 0) { res.numMisReplicatedBlocks++; misReplicatedPerFile++; if (!showFiles) { @@ -387,7 +385,9 @@ public class NamenodeFsck { out.print(path + ": "); } out.println(" Replica placement policy is violated for " + - block + ". " + blockPlacementStatus.getErrorDescription()); + block + + ". Block should be additionally replicated on " + + missingRacks + " more rack(s)."); } report.append(i + ". " + blkName + " len=" + block.getNumBytes()); if (locs.length == 0) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index bcebce4e201..9aaeb74a1c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -83,6 +83,7 @@ import org.apache.log4j.RollingFileAppender; import org.junit.Test; import com.google.common.collect.Sets; +import org.mockito.Mockito; import static org.mockito.Mockito.*; /** @@ -891,80 +892,6 @@ public class TestFsck { } } } - - /** - * Tests that the # of misreplaced replicas is correct - * @throws IOException - */ - @Test - public void testFsckMisPlacedReplicas() throws IOException { - // Desired replication factor - final short REPL_FACTOR = 2; - // Number of replicas to actually start - short NUM_DN = 2; - // Number of blocks to write - final short NUM_BLOCKS = 3; - // Set a small-ish blocksize - final long blockSize = 512; - - String [] racks = {"/rack1", "/rack1"}; - String [] hosts = {"host1", "host2"}; - - Configuration conf = new Configuration(); - conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); - - MiniDFSCluster cluster = null; - DistributedFileSystem dfs = null; - - try { - // Startup a minicluster - cluster = - new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts) - .racks(racks).build(); - assertNotNull("Failed Cluster Creation", cluster); - cluster.waitClusterUp(); - dfs = (DistributedFileSystem) cluster.getFileSystem(); - assertNotNull("Failed to get FileSystem", dfs); - - // Create a file that will be intentionally under-replicated - final String pathString = new String("/testfile"); - final Path path = new Path(pathString); - long fileLen = blockSize * NUM_BLOCKS; - DFSTestUtil.createFile(dfs, path, fileLen, REPL_FACTOR, 1); - - // Create an under-replicated file - NameNode namenode = cluster.getNameNode(); - NetworkTopology nettop = cluster.getNamesystem().getBlockManager() - .getDatanodeManager().getNetworkTopology(); - // Add a new node on different rack, so previous blocks' replicas - // are considered to be misplaced - nettop.add(DFSTestUtil.getDatanodeDescriptor("/rack2", "/host3")); - NUM_DN++; - - Map pmap = new HashMap(); - Writer result = new StringWriter(); - PrintWriter out = new PrintWriter(result, true); - InetAddress remoteAddress = InetAddress.getLocalHost(); - NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out, - NUM_DN, (short)REPL_FACTOR, remoteAddress); - - // Run the fsck and check the Result - final HdfsFileStatus file = - namenode.getRpcServer().getFileInfo(pathString); - assertNotNull(file); - Result res = new Result(conf); - fsck.check(pathString, file, res); - // check misReplicatedBlock number. - assertEquals(res.numMisReplicatedBlocks, NUM_BLOCKS); - } finally { - if(dfs != null) { - dfs.close(); - } - if(cluster != null) { - cluster.shutdown(); - } - } - } /** Test fsck with FileNotFound */ @Test From 0ebab3a88a5f172a1180f4e88a91cf6194b273ca Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 22 Oct 2013 02:22:17 +0000 Subject: [PATCH 19/76] HDFS-5377. Heartbeats from Datandode should include one storage report per storage directory git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1534464 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 +++ .../hdfs/server/datanode/BPServiceActor.java | 16 +++++++------- .../datanode/fsdataset/FsDatasetSpi.java | 5 +++++ .../fsdataset/impl/FsDatasetImpl.java | 21 +++++++++++++++++++ .../server/datanode/SimulatedFSDataset.java | 6 ++++++ 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 3b975f242b5..ff318f291fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -36,3 +36,6 @@ IMPROVEMENTS: HDFS-4988. Datanode must support all the volumes as individual storages. (Arpit Agarwal) + + HDFS-5377. Heartbeats from Datandode should include one storage report + per storage directory (Arpit Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 62b9b7a7a80..32e32966f79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -449,17 +449,15 @@ class BPServiceActor implements Runnable { HeartbeatResponse sendHeartBeat() throws IOException { + StorageReport[] reports = + dn.getFSDataset().getStorageReports(bpos.getBlockPoolId()); if (LOG.isDebugEnabled()) { - LOG.debug("Sending heartbeat from service actor: " + this); + LOG.debug("Sending heartbeat with " + reports.length + + " storage reports from service actor: " + this); } - // reports number of failed volumes - StorageReport[] report = { new StorageReport(bpRegistration.getDatanodeUuid(), - false, - dn.getFSDataset().getCapacity(), - dn.getFSDataset().getDfsUsed(), - dn.getFSDataset().getRemaining(), - dn.getFSDataset().getBlockPoolUsed(bpos.getBlockPoolId())) }; - return bpNamenode.sendHeartbeat(bpRegistration, report, + + return bpNamenode.sendHeartbeat(bpRegistration, + reports, dn.getXmitsInProgress(), dn.getXceiverCount(), dn.getFSDataset().getNumFailedVolumes()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index d9f7655dd6f..f4c49bc04e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory; import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.ReflectionUtils; @@ -86,6 +87,10 @@ public interface FsDatasetSpi extends FSDatasetMBean { /** @return a list of volumes. */ public List getVolumes(); + /** @return one or more storage reports for attached volumes. */ + public StorageReport[] getStorageReports(String bpid) + throws IOException; + /** @return the volume that contains a replica of the block. */ public V getVolume(ExtendedBlock b); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 2d84e6ac843..85238e459bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy; import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.util.DataChecksum; @@ -109,6 +110,26 @@ class FsDatasetImpl implements FsDatasetSpi { return volumes.volumes; } + @Override // FsDatasetSpi + public StorageReport[] getStorageReports(String bpid) + throws IOException { + StorageReport[] reports; + synchronized (statsLock) { + reports = new StorageReport[volumes.volumes.size()]; + int i = 0; + for (FsVolumeImpl volume : volumes.volumes) { + reports[i++] = new StorageReport(volume.getStorageID(), + false, + volume.getCapacity(), + volume.getDfsUsed(), + volume.getAvailable(), + volume.getBlockPoolUsed(bpid)); + } + } + + return reports; + } + @Override public synchronized FsVolumeImpl getVolume(final ExtendedBlock b) { final ReplicaInfo r = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index c0a9481c249..05033d6ff5c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.RollingLogs; import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.metrics2.util.MBeans; import org.apache.hadoop.util.DataChecksum; @@ -987,6 +988,11 @@ public class SimulatedFSDataset implements FsDatasetSpi { throw new UnsupportedOperationException(); } + @Override + public StorageReport[] getStorageReports(String bpid) { + throw new UnsupportedOperationException(); + } + @Override public List getFinalizedBlocks(String bpid) { throw new UnsupportedOperationException(); From b884af72c563403a7264b1a4b42ab58790af72f3 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 22 Oct 2013 23:29:40 +0000 Subject: [PATCH 20/76] HDFS-5398. NameNode changes to process storage reports per storage directory. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1534847 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 7 +- .../server/blockmanagement/BlockManager.java | 9 ++- .../blockmanagement/DatanodeDescriptor.java | 76 +++++++++---------- .../blockmanagement/DatanodeManager.java | 7 +- .../blockmanagement/DatanodeStorageInfo.java | 15 +++- .../blockmanagement/HeartbeatManager.java | 9 +-- .../hdfs/server/namenode/FSNamesystem.java | 8 +- .../server/namenode/NameNodeRpcServer.java | 6 +- .../hdfs/server/protocol/StorageReport.java | 2 + .../blockmanagement/BlockManagerTestUtil.java | 17 +++++ .../blockmanagement/TestBlockManager.java | 6 +- .../TestOverReplicatedBlocks.java | 5 +- .../TestReplicationPolicy.java | 28 ++++--- .../TestReplicationPolicyWithNodeGroup.java | 33 +++++--- .../hdfs/server/common/TestJspHelper.java | 6 +- .../hdfs/server/namenode/NameNodeAdapter.java | 5 +- 16 files changed, 147 insertions(+), 92 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index ff318f291fd..edc6bee4fe0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -4,7 +4,7 @@ IMPROVEMENTS: HDFS-4985. Add storage type to the protocol and expose it in block report and block locations. (Arpit Agarwal) - + HDFS-5115. Make StorageID a UUID. (Arpit Agarwal) HDFS-5000. DataNode configuration should allow specifying storage type. @@ -38,4 +38,7 @@ IMPROVEMENTS: (Arpit Agarwal) HDFS-5377. Heartbeats from Datandode should include one storage report - per storage directory (Arpit Agarwal) + per storage directory. (Arpit Agarwal) + + HDFS-5398. NameNode changes to process storage reports per storage + directory. (Arpit Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 38525d165b3..86bc20cd6bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2628,7 +2628,14 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block // Decrement number of blocks scheduled to this storage. // for a retry request (of DatanodeProtocol#blockReceivedAndDeleted with // RECEIVED_BLOCK), we currently also decrease the approximate number. - node.getStorageInfo(storageID).decrementBlocksScheduled(); + DatanodeStorageInfo storageInfo = node.getStorageInfo(storageID); + if (storageInfo != null) { + storageInfo.decrementBlocksScheduled(); + } else { + throw new IllegalArgumentException( + "Unrecognized storageID " + storageID + " in block report " + + "from Datanode " + node.toString()); + } // get the deletion hint node DatanodeDescriptor delHintNode = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index d2343bcc400..af1ef2dec23 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -27,12 +27,16 @@ import java.util.List; import java.util.Map; import java.util.Queue; +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.util.Time; @@ -44,6 +48,7 @@ import org.apache.hadoop.util.Time; @InterfaceAudience.Private @InterfaceStability.Evolving public class DatanodeDescriptor extends DatanodeInfo { + public static final Log LOG = LogFactory.getLog(DatanodeDescriptor.class); public static final DatanodeDescriptor[] EMPTY_ARRAY = {}; // Stores status of decommissioning. @@ -138,7 +143,7 @@ public class DatanodeDescriptor extends DatanodeInfo { * @param nodeID id of the data node */ public DatanodeDescriptor(DatanodeID nodeID) { - this(nodeID, 0L, 0L, 0L, 0L, 0, 0); + super(nodeID); } /** @@ -148,51 +153,21 @@ public class DatanodeDescriptor extends DatanodeInfo { */ public DatanodeDescriptor(DatanodeID nodeID, String networkLocation) { - this(nodeID, networkLocation, 0L, 0L, 0L, 0L, 0, 0); - } - - /** - * DatanodeDescriptor constructor - * @param nodeID id of the data node - * @param capacity capacity of the data node - * @param dfsUsed space used by the data node - * @param remaining remaining capacity of the data node - * @param bpused space used by the block pool corresponding to this namenode - * @param xceiverCount # of data transfers at the data node - */ - public DatanodeDescriptor(DatanodeID nodeID, - long capacity, - long dfsUsed, - long remaining, - long bpused, - int xceiverCount, - int failedVolumes) { - super(nodeID); - updateHeartbeat(capacity, dfsUsed, remaining, bpused, xceiverCount, - failedVolumes); + this(nodeID, networkLocation, 0, 0); } /** * DatanodeDescriptor constructor * @param nodeID id of the data node * @param networkLocation location of the data node in network - * @param capacity capacity of the data node, including space used by non-dfs - * @param dfsUsed the used space by dfs datanode - * @param remaining remaining capacity of the data node - * @param bpused space used by the block pool corresponding to this namenode * @param xceiverCount # of data transfers at the data node */ public DatanodeDescriptor(DatanodeID nodeID, String networkLocation, - long capacity, - long dfsUsed, - long remaining, - long bpused, int xceiverCount, int failedVolumes) { super(nodeID, networkLocation); - updateHeartbeat(capacity, dfsUsed, remaining, bpused, xceiverCount, - failedVolumes); + updateHeartbeat(StorageReport.EMPTY_ARRAY, xceiverCount, failedVolumes); } /** @@ -294,18 +269,37 @@ public class DatanodeDescriptor extends DatanodeInfo { /** * Updates stats from datanode heartbeat. */ - public void updateHeartbeat(long capacity, long dfsUsed, long remaining, - long blockPoolUsed, int xceiverCount, int volFailures) { - setCapacity(capacity); - setRemaining(remaining); - setBlockPoolUsed(blockPoolUsed); - setDfsUsed(dfsUsed); + public void updateHeartbeat(StorageReport[] reports, int xceiverCount, + int volFailures) { + long totalCapacity = 0; + long totalRemaining = 0; + long totalBlockPoolUsed = 0; + long totalDfsUsed = 0; + setXceiverCount(xceiverCount); setLastUpdate(Time.now()); this.volumeFailures = volFailures; - for(DatanodeStorageInfo storage : getStorageInfos()) { - storage.receivedHeartbeat(getLastUpdate()); + for (StorageReport report : reports) { + DatanodeStorageInfo storage = storageMap.get(report.getStorageID()); + if (storage != null) { + storage.receivedHeartbeat(report, getLastUpdate()); + totalCapacity += report.getCapacity(); + totalRemaining += report.getRemaining(); + totalBlockPoolUsed += report.getBlockPoolUsed(); + totalDfsUsed += report.getDfsUsed(); + } else { + // This warning is generally benign during cluster initialization + // when the heartbeat is received before the initial block reports + // from each storage. + LOG.warn("Unrecognized storage ID " + report.getStorageID()); + } } + + // Update total metrics for the node. + setCapacity(totalCapacity); + setRemaining(totalRemaining); + setBlockPoolUsed(totalBlockPoolUsed); + setDfsUsed(totalDfsUsed); } private static class BlockIterator implements Iterator { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 2d97941d095..93866e94987 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -1215,8 +1215,7 @@ public class DatanodeManager { /** Handle heartbeat from datanodes. */ public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg, - final String blockPoolId, - long capacity, long dfsUsed, long remaining, long blockPoolUsed, + StorageReport[] reports, final String blockPoolId, int xceiverCount, int maxTransfers, int failedVolumes ) throws IOException { synchronized (heartbeatManager) { @@ -1238,8 +1237,8 @@ public class DatanodeManager { return new DatanodeCommand[]{RegisterCommand.REGISTER}; } - heartbeatManager.updateHeartbeat(nodeinfo, capacity, dfsUsed, - remaining, blockPoolUsed, xceiverCount, failedVolumes); + heartbeatManager.updateHeartbeat(nodeinfo, reports, + xceiverCount, failedVolumes); // If we are in safemode, do not send back any recovery / replication // requests. Don't even drain the existing queue of work. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index af07e9c2c17..e5f4e8b21d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -21,6 +21,7 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; @@ -96,6 +97,7 @@ public class DatanodeStorageInfo { private long capacity; private long dfsUsed; private long remaining; + private long blockPoolUsed; private volatile BlockInfo blockList = null; private int numBlocks = 0; @@ -153,7 +155,8 @@ public class DatanodeStorageInfo { blockContentsStale = true; } - void receivedHeartbeat(final long lastUpdate) { + void receivedHeartbeat(StorageReport report, final long lastUpdate) { + updateState(report); heartbeatedSinceFailover = true; rollBlocksScheduled(lastUpdate); } @@ -165,10 +168,13 @@ public class DatanodeStorageInfo { blockReportCount++; } - void setUtilization(long capacity, long dfsUsed, long remaining) { + @VisibleForTesting + public void setUtilization(long capacity, long dfsUsed, + long remaining, long blockPoolUsed) { this.capacity = capacity; this.dfsUsed = dfsUsed; this.remaining = remaining; + this.blockPoolUsed = blockPoolUsed; } public void setState(State s) { @@ -201,6 +207,10 @@ public class DatanodeStorageInfo { return remaining; } + public long getBlockPoolUsed() { + return blockPoolUsed; + } + public boolean addBlock(BlockInfo b) { if(!b.addStorage(this)) return false; @@ -232,6 +242,7 @@ public class DatanodeStorageInfo { capacity = r.getCapacity(); dfsUsed = r.getDfsUsed(); remaining = r.getRemaining(); + blockPoolUsed = r.getBlockPoolUsed(); } public DatanodeDescriptor getDatanodeDescriptor() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java index 0bff1bf52f7..c7aba24c21e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.server.namenode.Namesystem; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.Time; @@ -170,7 +171,7 @@ class HeartbeatManager implements DatanodeStatistics { addDatanode(d); //update its timestamp - d.updateHeartbeat(0L, 0L, 0L, 0L, 0, 0); + d.updateHeartbeat(StorageReport.EMPTY_ARRAY, 0, 0); } } @@ -192,11 +193,9 @@ class HeartbeatManager implements DatanodeStatistics { } synchronized void updateHeartbeat(final DatanodeDescriptor node, - long capacity, long dfsUsed, long remaining, long blockPoolUsed, - int xceiverCount, int failedVolumes) { + StorageReport[] reports, int xceiverCount, int failedVolumes) { stats.subtract(node); - node.updateHeartbeat(capacity, dfsUsed, remaining, blockPoolUsed, - xceiverCount, failedVolumes); + node.updateHeartbeat(reports, xceiverCount, failedVolumes); stats.add(node); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 756397ec7ac..c13139d37c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -205,6 +205,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.util.ChunkedArrayList; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; @@ -4047,16 +4048,15 @@ public class FSNamesystem implements Namesystem, FSClusterStats, * @throws IOException */ HeartbeatResponse handleHeartbeat(DatanodeRegistration nodeReg, - long capacity, long dfsUsed, long remaining, long blockPoolUsed, - int xceiverCount, int xmitsInProgress, int failedVolumes) + StorageReport[] reports, int xceiverCount, int xmitsInProgress, + int failedVolumes) throws IOException { readLock(); try { final int maxTransfer = blockManager.getMaxReplicationStreams() - xmitsInProgress; DatanodeCommand[] cmds = blockManager.getDatanodeManager().handleHeartbeat( - nodeReg, blockPoolId, capacity, dfsUsed, remaining, blockPoolUsed, - xceiverCount, maxTransfer, failedVolumes); + nodeReg, reports, blockPoolId, xceiverCount, maxTransfer, failedVolumes); return new HeartbeatResponse(cmds, createHaStatusHeartbeat()); } finally { readUnlock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 4c2b5af68b1..9c92ecc8180 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -960,10 +960,8 @@ class NameNodeRpcServer implements NamenodeProtocols { StorageReport[] report, int xmitsInProgress, int xceiverCount, int failedVolumes) throws IOException { verifyRequest(nodeReg); - return namesystem.handleHeartbeat(nodeReg, report[0].getCapacity(), - report[0].getDfsUsed(), report[0].getRemaining(), - report[0].getBlockPoolUsed(), xceiverCount, xmitsInProgress, - failedVolumes); + return namesystem.handleHeartbeat(nodeReg, report, xceiverCount, + xmitsInProgress, failedVolumes); } @Override // DatanodeProtocol diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java index d018def4e21..c805f1ea455 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java @@ -27,6 +27,8 @@ public class StorageReport { private final long dfsUsed; private final long remaining; private final long blockPoolUsed; + + public static final StorageReport[] EMPTY_ARRAY = {}; public StorageReport(String sid, boolean failed, long capacity, long dfsUsed, long remaining, long bpUsed) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java index 8740b8471d6..41dbbcf5012 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java @@ -18,16 +18,19 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.Set; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.util.Daemon; import org.junit.Assert; @@ -240,4 +243,18 @@ public class BlockManagerTestUtil { } return dn; } + + + public static StorageReport[] getStorageReportsForDatanode( + DatanodeDescriptor dnd) { + ArrayList reports = new ArrayList(); + for (DatanodeStorageInfo storage : dnd.getStorageInfos()) { + StorageReport report = new StorageReport( + storage.getStorageID(), false, storage.getCapacity(), + storage.getDfsUsed(), storage.getRemaining(), + storage.getBlockPoolUsed()); + reports.add(report); + } + return reports.toArray(StorageReport.EMPTY_ARRAY); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 6878eecc3b4..71b09fda02e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -103,9 +103,11 @@ public class TestBlockManager { // construct network topology for (DatanodeDescriptor dn : nodesToAdd) { cluster.add(dn); + dn.getStorageInfos()[0].setUtilization( + 2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, + 2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L); dn.updateHeartbeat( - 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, - 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0); + BlockManagerTestUtil.getStorageReportsForDatanode(dn), 0, 0); bm.getDatanodeManager().checkIfClusterIsNowMultiRack(dn); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java index 412dd611de5..d27b02cfac2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java @@ -103,7 +103,10 @@ public class TestOverReplicatedBlocks { String corruptMachineName = corruptDataNode.getXferAddr(); for (DatanodeDescriptor datanode : hm.getDatanodes()) { if (!corruptMachineName.equals(datanode.getXferAddr())) { - datanode.updateHeartbeat(100L, 100L, 0L, 100L, 0, 0); + datanode.getStorageInfos()[0].setUtilization(100L, 100L, 0, 100L); + datanode.updateHeartbeat( + BlockManagerTestUtil.getStorageReportsForDatanode(datanode), + 0, 0); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index 21dd5f04262..3942f5d0803 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -90,6 +90,16 @@ public class TestReplicationPolicy { @Rule public ExpectedException exception = ExpectedException.none(); + private static void updateHeartbeatWithUsage(DatanodeDescriptor dn, + long capacity, long dfsUsed, long remaining, long blockPoolUsed, + int xceiverCount, int volFailures) { + dn.getStorageInfos()[0].setUtilization( + capacity, dfsUsed, remaining, blockPoolUsed); + dn.updateHeartbeat( + BlockManagerTestUtil.getStorageReportsForDatanode(dn), + xceiverCount, volFailures); + } + @BeforeClass public static void setupCluster() throws Exception { Configuration conf = new HdfsConfiguration(); @@ -126,7 +136,7 @@ public class TestReplicationPolicy { dataNodes[i]); } for (int i=0; i < NUM_OF_DATANODES; i++) { - dataNodes[i].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[i], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0); } @@ -150,7 +160,7 @@ public class TestReplicationPolicy { */ @Test public void testChooseTarget1() throws Exception { - dataNodes[0].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[0], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 4, 0); // overloaded @@ -180,7 +190,7 @@ public class TestReplicationPolicy { isOnSameRack(targets[2], targets[3])); assertFalse(isOnSameRack(targets[0], targets[2])); - dataNodes[0].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[0], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0); } @@ -303,7 +313,7 @@ public class TestReplicationPolicy { @Test public void testChooseTarget3() throws Exception { // make data node 0 to be not qualified to choose - dataNodes[0].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[0], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); // no space @@ -336,7 +346,7 @@ public class TestReplicationPolicy { isOnSameRack(targets[2], targets[3])); assertFalse(isOnSameRack(targets[1], targets[3])); - dataNodes[0].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[0], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0); } @@ -353,7 +363,7 @@ public class TestReplicationPolicy { public void testChoooseTarget4() throws Exception { // make data node 0 & 1 to be not qualified to choose: not enough disk space for(int i=0; i<2; i++) { - dataNodes[i].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[i], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); } @@ -381,7 +391,7 @@ public class TestReplicationPolicy { assertFalse(isOnSameRack(targets[0], targets[2])); for(int i=0; i<2; i++) { - dataNodes[i].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[i], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0); } @@ -443,7 +453,7 @@ public class TestReplicationPolicy { public void testChooseTargetWithMoreThanAvailableNodes() throws Exception { // make data node 0 & 1 to be not qualified to choose: not enough disk space for(int i=0; i<2; i++) { - dataNodes[i].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[i], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); } @@ -468,7 +478,7 @@ public class TestReplicationPolicy { assertTrue(((String)lastLogEntry.getMessage()).contains("in need of 2")); for(int i=0; i<2; i++) { - dataNodes[i].updateHeartbeat( + updateHeartbeatWithUsage(dataNodes[i], 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0, 0); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java index f38b4de21d2..15fc4a7e20e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java @@ -146,9 +146,20 @@ public class TestReplicationPolicyWithNodeGroup { namenode.stop(); } + private static void updateHeartbeatWithUsage(DatanodeDescriptor dn, + long capacity, long dfsUsed, long remaining, long blockPoolUsed, + int xceiverCount, int volFailures) { + dn.getStorageInfos()[0].setUtilization( + capacity, dfsUsed, remaining, blockPoolUsed); + dn.updateHeartbeat( + BlockManagerTestUtil.getStorageReportsForDatanode(dn), + xceiverCount, volFailures); + } + + private static void setupDataNodeCapacity() { for(int i=0; i live = new ArrayList(); live.add(dnDesc1); live.add(dnDesc2); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java index cf64c335bac..df434f6d405 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.MkdirOp; @@ -110,8 +111,8 @@ public class NameNodeAdapter { public static HeartbeatResponse sendHeartBeat(DatanodeRegistration nodeReg, DatanodeDescriptor dd, FSNamesystem namesystem) throws IOException { - return namesystem.handleHeartbeat(nodeReg, dd.getCapacity(), - dd.getDfsUsed(), dd.getRemaining(), dd.getBlockPoolUsed(), 0, 0, 0); + return namesystem.handleHeartbeat(nodeReg, + BlockManagerTestUtil.getStorageReportsForDatanode(dd), 0, 0, 0); } public static boolean setReplication(final FSNamesystem ns, From 01f37e42f050207b7659bf74e2484cf8bdae2d89 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 23 Oct 2013 01:28:48 +0000 Subject: [PATCH 21/76] HDFS-5390. Send one incremental block report per storage directory. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1534891 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 + .../hdfs/server/datanode/BPOfferService.java | 13 +- .../hdfs/server/datanode/BPServiceActor.java | 179 ++++++++++++------ .../hdfs/server/datanode/BlockReceiver.java | 16 +- .../hadoop/hdfs/server/datanode/DataNode.java | 20 +- .../hdfs/server/datanode/DataXceiver.java | 11 +- .../hadoop/hdfs/server/datanode/Replica.java | 5 + .../hdfs/server/datanode/ReplicaInfo.java | 8 + .../datanode/fsdataset/FsDatasetSpi.java | 2 +- .../datanode/fsdataset/FsVolumeSpi.java | 3 + .../impl/FsDatasetAsyncDiskService.java | 2 +- .../fsdataset/impl/FsDatasetImpl.java | 3 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 1 + .../server/datanode/SimulatedFSDataset.java | 14 +- .../server/datanode/TestBPOfferService.java | 2 +- .../server/datanode/TestDirectoryScanner.java | 5 + 16 files changed, 204 insertions(+), 83 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index edc6bee4fe0..80b8ba800d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -42,3 +42,6 @@ IMPROVEMENTS: HDFS-5398. NameNode changes to process storage reports per storage directory. (Arpit Agarwal) + + HDFS-5390. Send one incremental block report per storage directory. + (Arpit Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index e11cc87b4a8..c9486e9e4bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -192,7 +192,8 @@ class BPOfferService { * till namenode is informed before responding with success to the * client? For now we don't. */ - void notifyNamenodeReceivedBlock(ExtendedBlock block, String delHint) { + void notifyNamenodeReceivedBlock( + ExtendedBlock block, String delHint, String storageUuid) { checkBlock(block); checkDelHint(delHint); ReceivedDeletedBlockInfo bInfo = new ReceivedDeletedBlockInfo( @@ -201,7 +202,7 @@ class BPOfferService { delHint); for (BPServiceActor actor : bpServices) { - actor.notifyNamenodeBlockImmediately(bInfo); + actor.notifyNamenodeBlockImmediately(bInfo, storageUuid); } } @@ -218,23 +219,23 @@ class BPOfferService { "delHint is null"); } - void notifyNamenodeDeletedBlock(ExtendedBlock block) { + void notifyNamenodeDeletedBlock(ExtendedBlock block, String storageUuid) { checkBlock(block); ReceivedDeletedBlockInfo bInfo = new ReceivedDeletedBlockInfo( block.getLocalBlock(), BlockStatus.DELETED_BLOCK, null); for (BPServiceActor actor : bpServices) { - actor.notifyNamenodeDeletedBlock(bInfo); + actor.notifyNamenodeDeletedBlock(bInfo, storageUuid); } } - void notifyNamenodeReceivingBlock(ExtendedBlock block) { + void notifyNamenodeReceivingBlock(ExtendedBlock block, String storageUuid) { checkBlock(block); ReceivedDeletedBlockInfo bInfo = new ReceivedDeletedBlockInfo( block.getLocalBlock(), BlockStatus.RECEIVING_BLOCK, null); for (BPServiceActor actor : bpServices) { - actor.notifyNamenodeBlockImmediately(bInfo); + actor.notifyNamenodeBlockImmediately(bInfo, storageUuid); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 32e32966f79..4eb843d247f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketTimeoutException; import java.util.Collection; -import java.util.Iterator; import java.util.Map; import org.apache.commons.logging.Log; @@ -94,9 +93,9 @@ class BPServiceActor implements Runnable { * keyed by block ID, contains the pending changes which have yet to be * reported to the NN. Access should be synchronized on this object. */ - private final Map pendingIncrementalBR - = Maps.newHashMap(); - + private final Map + pendingIncrementalBRperStorage = Maps.newConcurrentMap(); + private volatile int pendingReceivedRequests = 0; private volatile boolean shouldServiceRun = true; private final DataNode dn; @@ -263,64 +262,84 @@ class BPServiceActor implements Runnable { * @throws IOException */ private void reportReceivedDeletedBlocks() throws IOException { - - // check if there are newly received blocks - ReceivedDeletedBlockInfo[] receivedAndDeletedBlockArray = null; - synchronized (pendingIncrementalBR) { - int numBlocks = pendingIncrementalBR.size(); - if (numBlocks > 0) { - // - // Send newly-received and deleted blockids to namenode - // - receivedAndDeletedBlockArray = pendingIncrementalBR - .values().toArray(new ReceivedDeletedBlockInfo[numBlocks]); + // For each storage, check if there are newly received blocks and if + // so then send an incremental report to the NameNode. + for (Map.Entry entry : + pendingIncrementalBRperStorage.entrySet()) { + final String storageUuid = entry.getKey(); + final PerStoragePendingIncrementalBR perStorageMap = entry.getValue(); + ReceivedDeletedBlockInfo[] receivedAndDeletedBlockArray = null; + // TODO: We can probably use finer-grained synchronization now. + synchronized (pendingIncrementalBRperStorage) { + if (perStorageMap.getBlockInfoCount() > 0) { + // Send newly-received and deleted blockids to namenode + receivedAndDeletedBlockArray = perStorageMap.dequeueBlockInfos(); + pendingReceivedRequests -= receivedAndDeletedBlockArray.length; + } } - pendingIncrementalBR.clear(); - } - if (receivedAndDeletedBlockArray != null) { - StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( - bpRegistration.getDatanodeUuid(), receivedAndDeletedBlockArray) }; - boolean success = false; - try { - bpNamenode.blockReceivedAndDeleted(bpRegistration, bpos.getBlockPoolId(), - report); - success = true; - } finally { - synchronized (pendingIncrementalBR) { - if (!success) { - // If we didn't succeed in sending the report, put all of the - // blocks back onto our queue, but only in the case where we didn't - // put something newer in the meantime. - for (ReceivedDeletedBlockInfo rdbi : receivedAndDeletedBlockArray) { - if (!pendingIncrementalBR.containsKey(rdbi.getBlock().getBlockId())) { - pendingIncrementalBR.put(rdbi.getBlock().getBlockId(), rdbi); - } + + if (receivedAndDeletedBlockArray != null) { + StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( + storageUuid, receivedAndDeletedBlockArray) }; + boolean success = false; + try { + bpNamenode.blockReceivedAndDeleted(bpRegistration, bpos.getBlockPoolId(), + report); + success = true; + } finally { + synchronized (pendingIncrementalBRperStorage) { + if (!success) { + // If we didn't succeed in sending the report, put all of the + // blocks back onto our queue, but only in the case where we + // didn't put something newer in the meantime. + perStorageMap.putMissingBlockInfos(receivedAndDeletedBlockArray); + pendingReceivedRequests += perStorageMap.getBlockInfoCount(); } } - pendingReceivedRequests = pendingIncrementalBR.size(); } } } } + /** + * Retrieve the incremental BR state for a given storage UUID + * @param storageUuid + * @return + */ + private PerStoragePendingIncrementalBR getIncrementalBRMapForStorage( + String storageUuid) { + PerStoragePendingIncrementalBR mapForStorage = + pendingIncrementalBRperStorage.get(storageUuid); + + if (mapForStorage == null) { + // This is the first time we are adding incremental BR state for + // this storage so create a new map. This is required once per + // storage, per service actor. + mapForStorage = new PerStoragePendingIncrementalBR(); + pendingIncrementalBRperStorage.put(storageUuid, mapForStorage); + } + + return mapForStorage; + } + /* * Informing the name node could take a long long time! Should we wait * till namenode is informed before responding with success to the * client? For now we don't. */ - void notifyNamenodeBlockImmediately(ReceivedDeletedBlockInfo bInfo) { - synchronized (pendingIncrementalBR) { - pendingIncrementalBR.put( - bInfo.getBlock().getBlockId(), bInfo); + void notifyNamenodeBlockImmediately( + ReceivedDeletedBlockInfo bInfo, String storageUuid) { + synchronized (pendingIncrementalBRperStorage) { + getIncrementalBRMapForStorage(storageUuid).putBlockInfo(bInfo); pendingReceivedRequests++; - pendingIncrementalBR.notifyAll(); + pendingIncrementalBRperStorage.notifyAll(); } } - void notifyNamenodeDeletedBlock(ReceivedDeletedBlockInfo bInfo) { - synchronized (pendingIncrementalBR) { - pendingIncrementalBR.put( - bInfo.getBlock().getBlockId(), bInfo); + void notifyNamenodeDeletedBlock( + ReceivedDeletedBlockInfo bInfo, String storageUuid) { + synchronized (pendingIncrementalBRperStorage) { + getIncrementalBRMapForStorage(storageUuid).putBlockInfo(bInfo); } } @@ -329,13 +348,13 @@ class BPServiceActor implements Runnable { */ @VisibleForTesting void triggerBlockReportForTests() { - synchronized (pendingIncrementalBR) { + synchronized (pendingIncrementalBRperStorage) { lastBlockReport = 0; lastHeartbeat = 0; - pendingIncrementalBR.notifyAll(); + pendingIncrementalBRperStorage.notifyAll(); while (lastBlockReport == 0) { try { - pendingIncrementalBR.wait(100); + pendingIncrementalBRperStorage.wait(100); } catch (InterruptedException e) { return; } @@ -345,12 +364,12 @@ class BPServiceActor implements Runnable { @VisibleForTesting void triggerHeartbeatForTests() { - synchronized (pendingIncrementalBR) { + synchronized (pendingIncrementalBRperStorage) { lastHeartbeat = 0; - pendingIncrementalBR.notifyAll(); + pendingIncrementalBRperStorage.notifyAll(); while (lastHeartbeat == 0) { try { - pendingIncrementalBR.wait(100); + pendingIncrementalBRperStorage.wait(100); } catch (InterruptedException e) { return; } @@ -360,13 +379,13 @@ class BPServiceActor implements Runnable { @VisibleForTesting void triggerDeletionReportForTests() { - synchronized (pendingIncrementalBR) { + synchronized (pendingIncrementalBRperStorage) { lastDeletedReport = 0; - pendingIncrementalBR.notifyAll(); + pendingIncrementalBRperStorage.notifyAll(); while (lastDeletedReport == 0) { try { - pendingIncrementalBR.wait(100); + pendingIncrementalBRperStorage.wait(100); } catch (InterruptedException e) { return; } @@ -582,10 +601,10 @@ class BPServiceActor implements Runnable { // long waitTime = dnConf.heartBeatInterval - (Time.now() - lastHeartbeat); - synchronized(pendingIncrementalBR) { + synchronized(pendingIncrementalBRperStorage) { if (waitTime > 0 && pendingReceivedRequests == 0) { try { - pendingIncrementalBR.wait(waitTime); + pendingIncrementalBRperStorage.wait(waitTime); } catch (InterruptedException ie) { LOG.warn("BPOfferService for " + this + " interrupted"); } @@ -756,4 +775,52 @@ class BPServiceActor implements Runnable { } } + private static class PerStoragePendingIncrementalBR { + private Map pendingIncrementalBR = + Maps.newHashMap(); + + /** + * Return the number of blocks on this storage that have pending + * incremental block reports. + * @return + */ + int getBlockInfoCount() { + return pendingIncrementalBR.size(); + } + + /** + * Dequeue and return all pending incremental block report state. + * @return + */ + ReceivedDeletedBlockInfo[] dequeueBlockInfos() { + ReceivedDeletedBlockInfo[] blockInfos = + pendingIncrementalBR.values().toArray( + new ReceivedDeletedBlockInfo[getBlockInfoCount()]); + + pendingIncrementalBR.clear(); + return blockInfos; + } + + /** + * Add blocks from blockArray to pendingIncrementalBR, unless the + * block already exists in pendingIncrementalBR. + * @param blockArray list of blocks to add. + */ + void putMissingBlockInfos(ReceivedDeletedBlockInfo[] blockArray) { + for (ReceivedDeletedBlockInfo rdbi : blockArray) { + if (!pendingIncrementalBR.containsKey(rdbi.getBlock().getBlockId())) { + pendingIncrementalBR.put(rdbi.getBlock().getBlockId(), rdbi); + } + } + } + + /** + * Add pending incremental block report for a single block. + * @param blockID + * @param blockInfo + */ + void putBlockInfo(ReceivedDeletedBlockInfo blockInfo) { + pendingIncrementalBR.put(blockInfo.getBlock().getBlockId(), blockInfo); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index 625daf06253..2dd4194114b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -162,7 +162,8 @@ class BlockReceiver implements Closeable { switch (stage) { case PIPELINE_SETUP_CREATE: replicaInfo = datanode.data.createRbw(block); - datanode.notifyNamenodeReceivingBlock(block); + datanode.notifyNamenodeReceivingBlock( + block, replicaInfo.getStorageUuid()); break; case PIPELINE_SETUP_STREAMING_RECOVERY: replicaInfo = datanode.data.recoverRbw( @@ -176,7 +177,8 @@ class BlockReceiver implements Closeable { block.getLocalBlock()); } block.setGenerationStamp(newGs); - datanode.notifyNamenodeReceivingBlock(block); + datanode.notifyNamenodeReceivingBlock( + block, replicaInfo.getStorageUuid()); break; case PIPELINE_SETUP_APPEND_RECOVERY: replicaInfo = datanode.data.recoverAppend(block, newGs, minBytesRcvd); @@ -185,7 +187,8 @@ class BlockReceiver implements Closeable { block.getLocalBlock()); } block.setGenerationStamp(newGs); - datanode.notifyNamenodeReceivingBlock(block); + datanode.notifyNamenodeReceivingBlock( + block, replicaInfo.getStorageUuid()); break; case TRANSFER_RBW: case TRANSFER_FINALIZED: @@ -252,6 +255,10 @@ class BlockReceiver implements Closeable { /** Return the datanode object. */ DataNode getDataNode() {return datanode;} + public Replica getReplicaInfo() { + return replicaInfo; + } + /** * close files. */ @@ -1072,7 +1079,8 @@ class BlockReceiver implements Closeable { : 0; block.setNumBytes(replicaInfo.getNumBytes()); datanode.data.finalizeBlock(block); - datanode.closeBlock(block, DataNode.EMPTY_DEL_HINT); + datanode.closeBlock( + block, DataNode.EMPTY_DEL_HINT, replicaInfo.getStorageUuid()); if (ClientTraceLog.isInfoEnabled() && isClient) { long offset = 0; DatanodeRegistration dnR = datanode.getDNRegistrationForBP(block diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index c051452aaca..526c89e7a43 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -520,10 +520,11 @@ public class DataNode extends Configured } // calls specific to BP - protected void notifyNamenodeReceivedBlock(ExtendedBlock block, String delHint) { + protected void notifyNamenodeReceivedBlock( + ExtendedBlock block, String delHint, String storageUuid) { BPOfferService bpos = blockPoolManager.get(block.getBlockPoolId()); if(bpos != null) { - bpos.notifyNamenodeReceivedBlock(block, delHint); + bpos.notifyNamenodeReceivedBlock(block, delHint, storageUuid); } else { LOG.error("Cannot find BPOfferService for reporting block received for bpid=" + block.getBlockPoolId()); @@ -531,10 +532,11 @@ public class DataNode extends Configured } // calls specific to BP - protected void notifyNamenodeReceivingBlock(ExtendedBlock block) { + protected void notifyNamenodeReceivingBlock( + ExtendedBlock block, String storageUuid) { BPOfferService bpos = blockPoolManager.get(block.getBlockPoolId()); if(bpos != null) { - bpos.notifyNamenodeReceivingBlock(block); + bpos.notifyNamenodeReceivingBlock(block, storageUuid); } else { LOG.error("Cannot find BPOfferService for reporting block receiving for bpid=" + block.getBlockPoolId()); @@ -542,10 +544,10 @@ public class DataNode extends Configured } /** Notify the corresponding namenode to delete the block. */ - public void notifyNamenodeDeletedBlock(ExtendedBlock block) { + public void notifyNamenodeDeletedBlock(ExtendedBlock block, String storageUuid) { BPOfferService bpos = blockPoolManager.get(block.getBlockPoolId()); if (bpos != null) { - bpos.notifyNamenodeDeletedBlock(block); + bpos.notifyNamenodeDeletedBlock(block, storageUuid); } else { LOG.error("Cannot find BPOfferService for reporting block deleted for bpid=" + block.getBlockPoolId()); @@ -1528,11 +1530,11 @@ public class DataNode extends Configured * @param block * @param delHint */ - void closeBlock(ExtendedBlock block, String delHint) { + void closeBlock(ExtendedBlock block, String delHint, String storageUuid) { metrics.incrBlocksWritten(); BPOfferService bpos = blockPoolManager.get(block.getBlockPoolId()); if(bpos != null) { - bpos.notifyNamenodeReceivedBlock(block, delHint); + bpos.notifyNamenodeReceivedBlock(block, delHint, storageUuid); } else { LOG.warn("Cannot find BPOfferService for reporting block received for bpid=" + block.getBlockPoolId()); @@ -1892,7 +1894,7 @@ public class DataNode extends Configured ExtendedBlock newBlock = new ExtendedBlock(oldBlock); newBlock.setGenerationStamp(recoveryId); newBlock.setNumBytes(newLength); - notifyNamenodeReceivedBlock(newBlock, ""); + notifyNamenodeReceivedBlock(newBlock, "", storageID); return storageID; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index e1de714d2ba..b4da358bda6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -447,6 +447,7 @@ class DataXceiver extends Receiver implements Runnable { String mirrorNode = null; // the name:port of next target String firstBadLink = ""; // first datanode that failed in connection setup Status mirrorInStatus = SUCCESS; + Replica replica; try { if (isDatanode || stage != BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) { @@ -457,8 +458,10 @@ class DataXceiver extends Receiver implements Runnable { stage, latestGenerationStamp, minBytesRcvd, maxBytesRcvd, clientname, srcDataNode, datanode, requestedChecksum, cachingStrategy); + replica = blockReceiver.getReplicaInfo(); } else { - datanode.data.recoverClose(block, latestGenerationStamp, minBytesRcvd); + replica = + datanode.data.recoverClose(block, latestGenerationStamp, minBytesRcvd); } // @@ -590,7 +593,8 @@ class DataXceiver extends Receiver implements Runnable { // the block is finalized in the PacketResponder. if (isDatanode || stage == BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) { - datanode.closeBlock(block, DataNode.EMPTY_DEL_HINT); + datanode.closeBlock( + block, DataNode.EMPTY_DEL_HINT, replica.getStorageUuid()); LOG.info("Received " + block + " src: " + remoteAddress + " dest: " + localAddress + " of size " + block.getNumBytes()); } @@ -859,7 +863,8 @@ class DataXceiver extends Receiver implements Runnable { dataXceiverServer.balanceThrottler, null); // notify name node - datanode.notifyNamenodeReceivedBlock(block, delHint); + datanode.notifyNamenodeReceivedBlock( + block, delHint, blockReceiver.getReplicaInfo().getStorageUuid()); LOG.info("Moved " + block + " from " + peer.getRemoteAddressString()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java index bd0485394a3..a480bb161f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/Replica.java @@ -54,4 +54,9 @@ public interface Replica { * @return the number of bytes that are visible to readers */ public long getVisibleLength(); + + /** + * Return the storageUuid of the volume that stores this replica. + */ + public String getStorageUuid(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java index 76bd1c0c6a4..27467f23ff3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java @@ -137,6 +137,14 @@ abstract public class ReplicaInfo extends Block implements Replica { void setVolume(FsVolumeSpi vol) { this.volume = vol; } + + /** + * Get the storageUuid of the volume that stores this replica. + */ + @Override + public String getStorageUuid() { + return volume.getStorageID(); + } /** * Return the parent directory path where this replica is located diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index f4c49bc04e8..3f8158b9a9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@ -243,7 +243,7 @@ public interface FsDatasetSpi extends FSDatasetMBean { * @param expectedBlockLen the number of bytes the replica is expected to have * @throws IOException */ - public void recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen + public Replica recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen ) throws IOException; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java index c0d71303876..b14ef562541 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java @@ -26,6 +26,9 @@ import org.apache.hadoop.hdfs.StorageType; * This is an interface for the underlying volume. */ public interface FsVolumeSpi { + /** @return the StorageUuid of the volume */ + public String getStorageID(); + /** @return a list of block pools. */ public String[] getBlockPoolList(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java index 806921d2263..4c0e19e612e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java @@ -195,7 +195,7 @@ class FsDatasetAsyncDiskService { + " at file " + blockFile + ". Ignored."); } else { if(block.getLocalBlock().getNumBytes() != BlockCommand.NO_ACK){ - datanode.notifyNamenodeDeletedBlock(block); + datanode.notifyNamenodeDeletedBlock(block, volume.getStorageID()); } volume.decDfsUsed(block.getBlockPoolId(), dfsBytes); LOG.info("Deleted " + block.getBlockPoolId() + " " diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 85238e459bf..f5928e47cdb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -699,7 +699,7 @@ class FsDatasetImpl implements FsDatasetSpi { } @Override // FsDatasetSpi - public void recoverClose(ExtendedBlock b, long newGS, + public Replica recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException { LOG.info("Recover failed close " + b); // check replica's state @@ -710,6 +710,7 @@ class FsDatasetImpl implements FsDatasetSpi { if (replicaInfo.getState() == ReplicaState.RBW) { finalizeReplica(b.getBlockPoolId(), replicaInfo); } + return replicaInfo; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index 30574b3b6d9..24eb5779070 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -290,6 +290,7 @@ class FsVolumeImpl implements FsVolumeSpi { } } + @Override public String getStorageID() { return storageID; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 05033d6ff5c..bf211afdcea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -131,6 +131,11 @@ public class SimulatedFSDataset implements FsDatasetSpi { } } + @Override + public String getStorageUuid() { + return storage.getStorageUuid(); + } + @Override synchronized public long getGenerationStamp() { return theBlock.getGenerationStamp(); @@ -314,6 +319,8 @@ public class SimulatedFSDataset implements FsDatasetSpi { private static class SimulatedStorage { private Map map = new HashMap(); + private final String storageUuid = "SimulatedStorage-UUID"; + private long capacity; // in bytes synchronized long getFree() { @@ -375,6 +382,10 @@ public class SimulatedFSDataset implements FsDatasetSpi { } return bpStorage; } + + public String getStorageUuid() { + return storageUuid; + } } private final Map> blockMap @@ -625,7 +636,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { } @Override // FsDatasetSpi - public void recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen) + public Replica recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException { final Map map = getMap(b.getBlockPoolId()); BInfo binfo = map.get(b.getLocalBlock()); @@ -639,6 +650,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { map.remove(b.getLocalBlock()); binfo.theBlock.setGenerationStamp(newGS); map.put(binfo.theBlock, binfo); + return binfo; } @Override // FsDatasetSpi diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java index 2b027004fe0..5374917fb70 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java @@ -176,7 +176,7 @@ public class TestBPOfferService { waitForBlockReport(mockNN2); // When we receive a block, it should report it to both NNs - bpos.notifyNamenodeReceivedBlock(FAKE_BLOCK, ""); + bpos.notifyNamenodeReceivedBlock(FAKE_BLOCK, "", ""); ReceivedDeletedBlockInfo[] ret = waitForBlockReceived(FAKE_BLOCK, mockNN1); assertEquals(1, ret.length); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java index cec8a15daa4..25182669f92 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java @@ -412,6 +412,11 @@ public class TestDirectoryScanner { public StorageType getStorageType() { return StorageType.DEFAULT; } + + @Override + public String getStorageID() { + return ""; + } } private final static TestFsVolumeSpi TEST_VOLUME = new TestFsVolumeSpi(); From f39f8c57344ede533ca4363c98230f3a0c401a76 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 23 Oct 2013 20:28:54 +0000 Subject: [PATCH 22/76] HDFS-5401. Fix NPE in Directory Scanner. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1535158 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 ++ .../hdfs/server/datanode/BPOfferService.java | 6 ++-- .../hdfs/server/datanode/BPServiceActor.java | 11 ++++-- .../hadoop/hdfs/server/datanode/DataNode.java | 8 +++-- .../fsdataset/impl/FsDatasetImpl.java | 35 ++++++++++--------- .../datanode/fsdataset/impl/FsVolumeList.java | 4 +-- 6 files changed, 42 insertions(+), 24 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 80b8ba800d6..cd139d4845e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -45,3 +45,5 @@ IMPROVEMENTS: HDFS-5390. Send one incremental block report per storage directory. (Arpit Agarwal) + + HDFS-5401. Fix NPE in Directory Scanner. (Arpit Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index c9486e9e4bb..5d584616df3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -27,6 +27,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import org.apache.commons.logging.Log; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -180,10 +181,11 @@ class BPOfferService { } } - void reportBadBlocks(ExtendedBlock block) { + void reportBadBlocks(ExtendedBlock block, + String storageUuid, StorageType storageType) { checkBlock(block); for (BPServiceActor actor : bpServices) { - actor.reportBadBlocks(block); + actor.reportBadBlocks(block, storageUuid, storageType); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 4eb843d247f..172fb0fc30e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -28,6 +28,7 @@ import java.util.Map; import org.apache.commons.logging.Log; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -237,12 +238,18 @@ class BPServiceActor implements Runnable { resetBlockReportTime = true; // reset future BRs for randomness } - void reportBadBlocks(ExtendedBlock block) { + void reportBadBlocks(ExtendedBlock block, + String storageUuid, StorageType storageType) { if (bpRegistration == null) { return; } DatanodeInfo[] dnArr = { new DatanodeInfo(bpRegistration) }; - LocatedBlock[] blocks = { new LocatedBlock(block, dnArr) }; + String[] uuids = { storageUuid }; + StorageType[] types = { storageType }; + // TODO: Corrupt flag is set to false for compatibility. We can probably + // set it to true here. + LocatedBlock[] blocks = { + new LocatedBlock(block, dnArr, uuids, types, -1, false) }; try { bpNamenode.reportBadBlocks(blocks); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 526c89e7a43..318d2f3705a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -559,7 +559,9 @@ public class DataNode extends Configured */ public void reportBadBlocks(ExtendedBlock block) throws IOException{ BPOfferService bpos = getBPOSForBlock(block); - bpos.reportBadBlocks(block); + FsVolumeSpi volume = getFSDataset().getVolume(block); + bpos.reportBadBlocks( + block, volume.getStorageID(), volume.getStorageType()); } /** @@ -1265,8 +1267,10 @@ public class DataNode extends Configured // Check if NN recorded length matches on-disk length long onDiskLength = data.getLength(block); if (block.getNumBytes() > onDiskLength) { + FsVolumeSpi volume = getFSDataset().getVolume(block); // Shorter on-disk len indicates corruption so report NN the corrupt block - bpos.reportBadBlocks(block); + bpos.reportBadBlocks( + block, volume.getStorageID(), volume.getStorageType()); LOG.warn("Can't replicate block " + block + " because on-disk length " + onDiskLength + " is shorter than NameNode recorded length " + block.getNumBytes()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index f5928e47cdb..9077c40a836 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -198,7 +198,9 @@ class FsDatasetImpl implements FsDatasetSpi { // two maps. This might require some refactoring // rewrite of FsDatasetImpl. final ReplicaMap volumeMap; - final Map perVolumeReplicaMap; + + // Map from StorageID to ReplicaMap. + final Map perVolumeReplicaMap; // Used for synchronizing access to usage stats @@ -249,7 +251,7 @@ class FsDatasetImpl implements FsDatasetSpi { LOG.info("Added volume - " + dir + ", StorageType: " + storageType); } volumeMap = new ReplicaMap(this); - perVolumeReplicaMap = new HashMap(); + perVolumeReplicaMap = new HashMap(); @SuppressWarnings("unchecked") final VolumeChoosingPolicy blockChooserImpl = @@ -628,7 +630,7 @@ class FsDatasetImpl implements FsDatasetSpi { // Replace finalized replica by a RBW replica in replicas map volumeMap.add(bpid, newReplicaInfo); - perVolumeReplicaMap.get(v).add(bpid, newReplicaInfo); + perVolumeReplicaMap.get(v.getStorageID()).add(bpid, newReplicaInfo); return newReplicaInfo; } @@ -759,7 +761,7 @@ class FsDatasetImpl implements FsDatasetSpi { ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(b.getBlockId(), b.getGenerationStamp(), v, f.getParentFile()); volumeMap.add(b.getBlockPoolId(), newReplicaInfo); - perVolumeReplicaMap.get(v).add(b.getBlockPoolId(), newReplicaInfo); + perVolumeReplicaMap.get(v.getStorageID()).add(b.getBlockPoolId(), newReplicaInfo); return newReplicaInfo; } @@ -878,7 +880,7 @@ class FsDatasetImpl implements FsDatasetSpi { rbw.setBytesAcked(visible); // overwrite the RBW in the volume map volumeMap.add(b.getBlockPoolId(), rbw); - perVolumeReplicaMap.get(v).add(b.getBlockPoolId(), rbw); + perVolumeReplicaMap.get(v.getStorageID()).add(b.getBlockPoolId(), rbw); return rbw; } @@ -898,7 +900,7 @@ class FsDatasetImpl implements FsDatasetSpi { ReplicaInPipeline newReplicaInfo = new ReplicaInPipeline(b.getBlockId(), b.getGenerationStamp(), v, f.getParentFile()); volumeMap.add(b.getBlockPoolId(), newReplicaInfo); - perVolumeReplicaMap.get(v).add(b.getBlockPoolId(), newReplicaInfo); + perVolumeReplicaMap.get(v.getStorageID()).add(b.getBlockPoolId(), newReplicaInfo); return newReplicaInfo; } @@ -967,7 +969,8 @@ class FsDatasetImpl implements FsDatasetSpi { newReplicaInfo = new FinalizedReplica(replicaInfo, v, dest.getParentFile()); } volumeMap.add(bpid, newReplicaInfo); - perVolumeReplicaMap.get(newReplicaInfo.getVolume()).add(bpid, newReplicaInfo); + perVolumeReplicaMap.get(newReplicaInfo.getVolume().getStorageID()) + .add(bpid, newReplicaInfo); return newReplicaInfo; } @@ -981,7 +984,7 @@ class FsDatasetImpl implements FsDatasetSpi { if (replicaInfo != null && replicaInfo.getState() == ReplicaState.TEMPORARY) { // remove from volumeMap volumeMap.remove(b.getBlockPoolId(), b.getLocalBlock()); - perVolumeReplicaMap.get((FsVolumeImpl) replicaInfo.getVolume()) + perVolumeReplicaMap.get(replicaInfo.getVolume().getStorageID()) .remove(b.getBlockPoolId(), b.getLocalBlock()); // delete the on-disk temp file @@ -1064,7 +1067,7 @@ class FsDatasetImpl implements FsDatasetSpi { new HashMap(); for (FsVolumeImpl v : getVolumes()) { - ReplicaMap rMap = perVolumeReplicaMap.get(v); + ReplicaMap rMap = perVolumeReplicaMap.get(v.getStorageID()); BlockListAsLongs blockList = getBlockReportWithReplicaMap(bpid, rMap); blockReportMap.put(v.getStorageID(), blockList); } @@ -1212,7 +1215,7 @@ class FsDatasetImpl implements FsDatasetSpi { v.clearPath(bpid, parent); } volumeMap.remove(bpid, invalidBlks[i]); - perVolumeReplicaMap.get(v).remove(bpid, invalidBlks[i]); + perVolumeReplicaMap.get(v.getStorageID()).remove(bpid, invalidBlks[i]); } // Delete the block asynchronously to make sure we can do it fast enough @@ -1274,7 +1277,8 @@ class FsDatasetImpl implements FsDatasetSpi { LOG.warn("Removing replica " + bpid + ":" + b.getBlockId() + " on failed volume " + fv.getCurrentDir().getAbsolutePath()); ib.remove(); - perVolumeReplicaMap.get(fv).remove(bpid, b.getBlockId()); + perVolumeReplicaMap.get(fv.getStorageID()) + .remove(bpid, b.getBlockId()); removedBlocks++; } } @@ -1391,8 +1395,7 @@ class FsDatasetImpl implements FsDatasetSpi { // Block is in memory and not on the disk // Remove the block from volumeMap volumeMap.remove(bpid, blockId); - perVolumeReplicaMap.get((FsVolumeImpl) memBlockInfo.getVolume()) - .remove(bpid, blockId); + perVolumeReplicaMap.get(vol.getStorageID()).remove(bpid, blockId); final DataBlockScanner blockScanner = datanode.getBlockScanner(); if (blockScanner != null) { blockScanner.deleteBlock(bpid, new Block(blockId)); @@ -1416,8 +1419,8 @@ class FsDatasetImpl implements FsDatasetSpi { ReplicaInfo diskBlockInfo = new FinalizedReplica(blockId, diskFile.length(), diskGS, vol, diskFile.getParentFile()); volumeMap.add(bpid, diskBlockInfo); - perVolumeReplicaMap.get((FsVolumeImpl) memBlockInfo.getVolume()). - remove(bpid, diskBlockInfo); + perVolumeReplicaMap.get(vol.getStorageID()) + .remove(bpid, diskBlockInfo); final DataBlockScanner blockScanner = datanode.getBlockScanner(); if (blockScanner != null) { blockScanner.addBlock(new ExtendedBlock(bpid, diskBlockInfo)); @@ -1695,7 +1698,7 @@ class FsDatasetImpl implements FsDatasetSpi { // TODO: Avoid the double scan. for (FsVolumeImpl v : getVolumes()) { - ReplicaMap rMap = perVolumeReplicaMap.get(v); + ReplicaMap rMap = perVolumeReplicaMap.get(v.getStorageID()); rMap.initBlockPool(bpid); volumes.getVolumeMap(bpid, v, rMap); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index a0e65db3647..671996718be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -90,13 +90,13 @@ class FsVolumeList { return remaining; } - void initializeReplicaMaps(Map perVolumeReplicaMap, + void initializeReplicaMaps(Map perVolumeReplicaMap, ReplicaMap globalReplicaMap, Object mutex) throws IOException { for (FsVolumeImpl v : volumes) { ReplicaMap rMap = new ReplicaMap(mutex); v.getVolumeMap(rMap); - perVolumeReplicaMap.put(v, rMap); + perVolumeReplicaMap.put(v.getStorageID(), rMap); globalReplicaMap.addAll(rMap); } } From ec01db59ce12746281938d8b0f331db72cd4c10a Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Fri, 25 Oct 2013 17:21:49 +0000 Subject: [PATCH 23/76] HDFS-5417. Fix storage IDs in PBHelper and UpgradeUtilities. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1535791 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 ++ .../org/apache/hadoop/hdfs/protocolPB/PBHelper.java | 11 ++++++++--- .../java/org/apache/hadoop/hdfs/UpgradeUtilities.java | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index cd139d4845e..d0235c760f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -47,3 +47,5 @@ IMPROVEMENTS: (Arpit Agarwal) HDFS-5401. Fix NPE in Directory Scanner. (Arpit Agarwal) + + HDFS-5417. Fix storage IDs in PBHelper and UpgradeUtilities. (szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index f1b3149e208..7cacb92a0d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -581,7 +581,10 @@ public class PBHelper { builder.addStorageTypes(PBHelper.convertStorageType(storageTypes[i])); } } - builder.addAllStorageIDs(Arrays.asList(b.getStorageIDs())); + final String[] storageIDs = b.getStorageIDs(); + if (storageIDs != null) { + builder.addAllStorageIDs(Arrays.asList(b.getStorageIDs())); + } return builder.setB(PBHelper.convert(b.getBlock())) .setBlockToken(PBHelper.convert(b.getBlockToken())) @@ -609,9 +612,11 @@ public class PBHelper { } } + final int storageIDsCount = proto.getStorageIDsCount(); + final String[] storageIDs = storageIDsCount == 0? null + : proto.getStorageIDsList().toArray(new String[storageIDsCount]); LocatedBlock lb = new LocatedBlock(PBHelper.convert(proto.getB()), targets, - proto.getStorageIDsList().toArray(new String[proto.getStorageIDsCount()]), - storageTypes, proto.getOffset(), proto.getCorrupt()); + storageIDs, storageTypes, proto.getOffset(), proto.getCorrupt()); lb.setBlockToken(PBHelper.convert(proto.getBlockToken())); return lb; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java index be111659255..52854588877 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java @@ -459,6 +459,7 @@ public class UpgradeUtilities { for (int i = 0; i < parent.length; i++) { File versionFile = new File(parent[i], "VERSION"); StorageDirectory sd = new StorageDirectory(parent[i].getParentFile()); + storage.createStorageID(sd); storage.writeProperties(versionFile, sd); versionFiles[i] = versionFile; File bpDir = BlockPoolSliceStorage.getBpRoot(bpid, parent[i]); From b9d561c548c26d0db4994e6c13c7ebf43705d794 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sun, 27 Oct 2013 18:29:49 +0000 Subject: [PATCH 24/76] HDFS-5214. Fix NPEs in BlockManager and DirectoryScanner. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1536179 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 + .../server/blockmanagement/BlockManager.java | 5 +- .../datanode/BlockPoolSliceScanner.java | 2 +- .../server/datanode/DirectoryScanner.java | 20 +-- .../server/datanode/FinalizedReplica.java | 4 + .../datanode/fsdataset/FsDatasetSpi.java | 3 +- .../fsdataset/impl/FsDatasetImpl.java | 7 +- .../server/datanode/SimulatedFSDataset.java | 2 +- .../hdfs/server/datanode/TestBlockReport.java | 163 ++++++++++++------ .../server/datanode/TestDirectoryScanner.java | 2 +- 10 files changed, 144 insertions(+), 66 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index d0235c760f9..d878b66b303 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -49,3 +49,5 @@ IMPROVEMENTS: HDFS-5401. Fix NPE in Directory Scanner. (Arpit Agarwal) HDFS-5417. Fix storage IDs in PBHelper and UpgradeUtilities. (szetszwo) + + HDFS-5214. Fix NPEs in BlockManager and DirectoryScanner. (Arpit Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 86bc20cd6bf..6d5c604ba7d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1833,7 +1833,10 @@ public class BlockManager { ReplicaState iState = itBR.getCurrentReplicaState(); BlockInfo storedBlock = processReportedBlock(dn, storage.getStorageID(), iblk, iState, toAdd, toInvalidate, toCorrupt, toUC); - toRemove.remove(storedBlock); + + if (storedBlock != null) { + toRemove.remove(storedBlock); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java index 32f9d9ea3e9..13a83bce5fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java @@ -187,7 +187,7 @@ class BlockPoolSliceScanner { + hours + " hours for block pool " + bpid); // get the list of blocks and arrange them in random order - List arr = dataset.getFinalizedBlocks(blockPoolId); + List arr = dataset.getFinalizedBlocks(blockPoolId); Collections.shuffle(arr); long scanTime = -1; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java index 1258f0e4981..17ec35d6fb6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java @@ -230,10 +230,6 @@ public class DirectoryScanner implements Runnable { throw new RuntimeException(prefix + " is not a prefix of " + fullPath); } - ScanInfo(long blockId) { - this(blockId, null, null, null); - } - ScanInfo(long blockId, File blockFile, File metaFile, FsVolumeSpi vol) { this.blockId = blockId; String condensedVolPath = vol == null ? null : @@ -439,8 +435,8 @@ public class DirectoryScanner implements Runnable { diffs.put(bpid, diffRecord); statsRecord.totalBlocks = blockpoolReport.length; - List bl = dataset.getFinalizedBlocks(bpid); - Block[] memReport = bl.toArray(new Block[bl.size()]); + List bl = dataset.getFinalizedBlocks(bpid); + FinalizedReplica[] memReport = bl.toArray(new FinalizedReplica[bl.size()]); Arrays.sort(memReport); // Sort based on blockId int d = 0; // index for blockpoolReport @@ -458,7 +454,8 @@ public class DirectoryScanner implements Runnable { } if (info.getBlockId() > memBlock.getBlockId()) { // Block is missing on the disk - addDifference(diffRecord, statsRecord, memBlock.getBlockId()); + addDifference(diffRecord, statsRecord, + memBlock.getBlockId(), info.getVolume()); m++; continue; } @@ -478,7 +475,9 @@ public class DirectoryScanner implements Runnable { m++; } while (m < memReport.length) { - addDifference(diffRecord, statsRecord, memReport[m++].getBlockId()); + FinalizedReplica current = memReport[m++]; + addDifference(diffRecord, statsRecord, + current.getBlockId(), current.getVolume()); } while (d < blockpoolReport.length) { statsRecord.missingMemoryBlocks++; @@ -502,10 +501,11 @@ public class DirectoryScanner implements Runnable { /** Block is not found on the disk */ private void addDifference(LinkedList diffRecord, - Stats statsRecord, long blockId) { + Stats statsRecord, long blockId, + FsVolumeSpi vol) { statsRecord.missingBlockFile++; statsRecord.missingMetaFile++; - diffRecord.add(new ScanInfo(blockId)); + diffRecord.add(new ScanInfo(blockId, null, null, vol)); } /** Is the given volume still valid in the dataset? */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java index 90658733ecd..1a852c34668 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java @@ -61,6 +61,10 @@ public class FinalizedReplica extends ReplicaInfo { this.unlinked = from.isUnlinked(); } + public FinalizedReplica(ReplicaInfo replicaInfo) { + super(replicaInfo); + } + @Override // ReplicaInfo public ReplicaState getState() { return ReplicaState.FINALIZED; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index 3f8158b9a9b..90edd5104ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataStorage; +import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica; import org.apache.hadoop.hdfs.server.datanode.Replica; import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory; @@ -98,7 +99,7 @@ public interface FsDatasetSpi extends FSDatasetMBean { public Map getVolumeInfoMap(); /** @return a list of finalized blocks for the given block pool. */ - public List getFinalizedBlocks(String bpid); + public List getFinalizedBlocks(String bpid); /** * Check whether the in-memory block record matches the block on the disk, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 9077c40a836..8677131d4ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -1079,11 +1079,12 @@ class FsDatasetImpl implements FsDatasetSpi { * Get the list of finalized blocks from in-memory blockmap for a block pool. */ @Override - public synchronized List getFinalizedBlocks(String bpid) { - ArrayList finalized = new ArrayList(volumeMap.size(bpid)); + public synchronized List getFinalizedBlocks(String bpid) { + ArrayList finalized = + new ArrayList(volumeMap.size(bpid)); for (ReplicaInfo b : volumeMap.replicas(bpid)) { if(b.getState() == ReplicaState.FINALIZED) { - finalized.add(new Block(b)); + finalized.add(new FinalizedReplica(b)); } } return finalized; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index bf211afdcea..6f3bed9fda0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -1006,7 +1006,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { } @Override - public List getFinalizedBlocks(String bpid) { + public List getFinalizedBlocks(String bpid) { throw new UnsupportedOperationException(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java index 0e60670a412..21d0339888a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java @@ -17,7 +17,9 @@ */ package org.apache.hadoop.hdfs.server.datanode; +import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import java.io.File; @@ -25,6 +27,7 @@ import java.io.FilenameFilter; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeoutException; @@ -89,7 +92,7 @@ public class TestBlockReport { private MiniDFSCluster cluster; private DistributedFileSystem fs; - Random rand = new Random(RAND_LIMIT); + private static Random rand = new Random(RAND_LIMIT); private static Configuration conf; @@ -113,6 +116,57 @@ public class TestBlockReport { cluster.shutdown(); } + private static StorageBlockReport[] getBlockReports(DataNode dn, String bpid) { + Map perVolumeBlockLists = + dn.getFSDataset().getBlockReports(bpid); + + // Send block report + StorageBlockReport[] reports = + new StorageBlockReport[perVolumeBlockLists.size()]; + + int i = 0; + for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { + String storageID = kvPair.getKey(); + long[] blockList = kvPair.getValue().getBlockListAsLongs(); + + // Dummy DatanodeStorage object just for sending the block report. + DatanodeStorage dnStorage = new DatanodeStorage(storageID); + reports[i++] = new StorageBlockReport(dnStorage, blockList); + } + + return reports; + } + + // Get block reports but modify the GS of one of the blocks. + private static StorageBlockReport[] getBlockReportsCorruptSingleBlockGS( + DataNode dn, String bpid) { + Map perVolumeBlockLists = + dn.getFSDataset().getBlockReports(bpid); + + // Send block report + StorageBlockReport[] reports = + new StorageBlockReport[perVolumeBlockLists.size()]; + + boolean corruptedBlock = false; + + int i = 0; + for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { + String storageID = kvPair.getKey(); + long[] blockList = kvPair.getValue().getBlockListAsLongs(); + + if (!corruptedBlock) { + blockList[4] = rand.nextInt(); // Bad GS. + corruptedBlock = true; + } + + // Dummy DatanodeStorage object just for sending the block report. + DatanodeStorage dnStorage = new DatanodeStorage(storageID); + reports[i++] = new StorageBlockReport(dnStorage, blockList); + } + + return reports; + } + /** * Test write a file, verifies and closes it. Then the length of the blocks * are messed up and BlockReport is forced. @@ -153,10 +207,8 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N0); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getDatanodeUuid()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn, poolId); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); List blocksAfterReport = DFSTestUtil.getAllBlocks(fs.open(filePath)); @@ -211,7 +263,6 @@ public class TestBlockReport { for (Integer aRemovedIndex : removedIndex) { blocks2Remove.add(lBlocks.get(aRemovedIndex).getBlock()); } - ArrayList blocks = locatedToBlocks(lBlocks, removedIndex); if(LOG.isDebugEnabled()) { LOG.debug("Number of blocks allocated " + lBlocks.size()); @@ -225,8 +276,11 @@ public class TestBlockReport { for (File f : findAllFiles(dataDir, new MyFileFilter(b.getBlockName(), true))) { DataNodeTestUtils.getFSDataset(dn0).unfinalizeBlock(b); - if (!f.delete()) + if (!f.delete()) { LOG.warn("Couldn't delete " + b.getBlockName()); + } else { + LOG.debug("Deleted file " + f.toString()); + } } } @@ -235,10 +289,8 @@ public class TestBlockReport { // all blocks belong to the same file, hence same BP String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn0.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getDatanodeUuid()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn0, poolId); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); BlockManagerTestUtil.getComputedDatanodeWork(cluster.getNamesystem() .getBlockManager()); @@ -253,9 +305,8 @@ public class TestBlockReport { /** - * Test writes a file and closes it. Then test finds a block - * and changes its GS to be < of original one. - * New empty block is added to the list of blocks. + * Test writes a file and closes it. + * Block reported is generated with a bad GS for a single block. * Block report is forced and the check for # of corrupted blocks is performed. * * @throws IOException in case of an error @@ -264,41 +315,65 @@ public class TestBlockReport { public void blockReport_03() throws IOException { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); - - ArrayList blocks = - prepareForRide(filePath, METHOD_NAME, FILE_SIZE); - - // The block with modified GS won't be found. Has to be deleted - blocks.get(0).setGenerationStamp(rand.nextLong()); - // This new block is unknown to NN and will be mark for deletion. - blocks.add(new Block()); + DFSTestUtil.createFile(fs, filePath, + FILE_SIZE, REPL_FACTOR, rand.nextLong()); // all blocks belong to the same file, hence same BP DataNode dn = cluster.getDataNodes().get(DN_N0); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getDatanodeUuid()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; + StorageBlockReport[] reports = getBlockReportsCorruptSingleBlockGS(dn, poolId); DatanodeCommand dnCmd = - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); if(LOG.isDebugEnabled()) { LOG.debug("Got the command: " + dnCmd); } printStats(); - assertEquals("Wrong number of CorruptedReplica+PendingDeletion " + - "blocks is found", 2, - cluster.getNamesystem().getCorruptReplicaBlocks() + - cluster.getNamesystem().getPendingDeletionBlocks()); + assertThat("Wrong number of corrupt blocks", + cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L)); + assertThat("Wrong number of PendingDeletion blocks", + cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); } /** - * This test isn't a representative case for BlockReport - * The empty method is going to be left here to keep the naming - * of the test plan in synch with the actual implementation + * Test writes a file and closes it. + * Block reported is generated with an extra block. + * Block report is forced and the check for # of pendingdeletion + * blocks is performed. + * + * @throws IOException in case of an error */ - public void blockReport_04() { + @Test + public void blockReport_04() throws IOException { + final String METHOD_NAME = GenericTestUtils.getMethodName(); + Path filePath = new Path("/" + METHOD_NAME + ".dat"); + DFSTestUtil.createFile(fs, filePath, + FILE_SIZE, REPL_FACTOR, rand.nextLong()); + + + DataNode dn = cluster.getDataNodes().get(DN_N0); + // all blocks belong to the same file, hence same BP + String poolId = cluster.getNamesystem().getBlockPoolId(); + + // Create a bogus new block which will not be present on the namenode. + ExtendedBlock b = new ExtendedBlock( + poolId, rand.nextLong(), 1024L, rand.nextLong()); + dn.getFSDataset().createRbw(b); + + DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); + StorageBlockReport[] reports = getBlockReports(dn, poolId); + DatanodeCommand dnCmd = + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); + if(LOG.isDebugEnabled()) { + LOG.debug("Got the command: " + dnCmd); + } + printStats(); + + assertThat("Wrong number of corrupt blocks", + cluster.getNamesystem().getCorruptReplicaBlocks(), is(0L)); + assertThat("Wrong number of PendingDeletion blocks", + cluster.getNamesystem().getPendingDeletionBlocks(), is(1L)); } // Client requests new block from NN. The test corrupts this very block @@ -331,10 +406,8 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getDatanodeUuid()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn, poolId); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); printStats(); assertEquals("Wrong number of PendingReplication Blocks", 0, cluster.getNamesystem().getUnderReplicatedBlocks()); @@ -382,9 +455,7 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getDatanodeUuid()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; + StorageBlockReport[] report = getBlockReports(dn, poolId); cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); assertEquals("Wrong number of Corrupted blocks", @@ -407,7 +478,7 @@ public class TestBlockReport { } report[0] = new StorageBlockReport( - new DatanodeStorage(dnR.getDatanodeUuid()), + report[0].getStorage(), new BlockListAsLongs(blocks, null).getBlockListAsLongs()); cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); @@ -458,9 +529,7 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getDatanodeUuid()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; + StorageBlockReport[] report = getBlockReports(dn, poolId); cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); assertEquals("Wrong number of PendingReplication blocks", @@ -506,9 +575,7 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = { new StorageBlockReport( - new DatanodeStorage(dnR.getDatanodeUuid()), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()) }; + StorageBlockReport[] report = getBlockReports(dn, poolId); cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); assertEquals("Wrong number of PendingReplication blocks", diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java index 25182669f92..f5b535d3943 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java @@ -447,7 +447,7 @@ public class TestDirectoryScanner { void testScanInfoObject(long blockId) throws Exception { DirectoryScanner.ScanInfo scanInfo = - new DirectoryScanner.ScanInfo(blockId); + new DirectoryScanner.ScanInfo(blockId, null, null, null); assertEquals(blockId, scanInfo.getBlockId()); assertNull(scanInfo.getBlockFile()); assertNull(scanInfo.getMetaFile()); From dc0b44a884700cda3665aa04b16d1e3474328e05 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 28 Oct 2013 17:29:06 +0000 Subject: [PATCH 25/76] HDFS-5435. File append fails to initialize storageIDs. (Junping Du) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1536434 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 +++ .../src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 2 +- .../hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index d878b66b303..b30e984db52 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -51,3 +51,6 @@ IMPROVEMENTS: HDFS-5417. Fix storage IDs in PBHelper and UpgradeUtilities. (szetszwo) HDFS-5214. Fix NPEs in BlockManager and DirectoryScanner. (Arpit Agarwal) + + HDFS-5435. File append fails to initialize storageIDs. (Junping Du via + Arpit Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index f9738e00e6a..9e91abad6e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -311,7 +311,6 @@ public class DFSOutputStream extends FSOutputSummer private DataInputStream blockReplyStream; private ResponseProcessor response = null; private volatile DatanodeInfo[] nodes = null; // list of targets for current block - //TODO: update storage IDs private volatile String[] storageIDs = null; private LoadingCache excludedNodes = CacheBuilder.newBuilder() @@ -404,6 +403,7 @@ public class DFSOutputStream extends FSOutputSummer // setup pipeline to append to the last block XXX retries?? nodes = lastBlock.getLocations(); + storageIDs = lastBlock.getStorageIDs(); errorIndex = -1; // no errors yet. if (nodes.length < 1) { throw new IOException("Unable to retrieve blocks locations " + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 30ea3e562e7..a58485be09f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -788,7 +788,7 @@ public class ClientNamenodeProtocolTranslatorPB implements .setOldBlock(PBHelper.convert(oldBlock)) .setNewBlock(PBHelper.convert(newBlock)) .addAllNewNodes(Arrays.asList(PBHelper.convert(newNodes))) - .addAllStorageIDs(Arrays.asList(storageIDs)) + .addAllStorageIDs(storageIDs == null ? null : Arrays.asList(storageIDs)) .build(); try { rpcProxy.updatePipeline(null, req); From 2af5083a35037fde1cfd34a8bfa6c8958cb65ff7 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 31 Oct 2013 02:32:25 +0000 Subject: [PATCH 26/76] HDFS-5437. Fix TestBlockReport and TestBPOfferService failures. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1537365 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 + .../hdfs/protocol/BlockListAsLongs.java | 41 ++-- .../hadoop/hdfs/protocol/LocatedBlock.java | 11 +- .../hdfs/server/datanode/BPServiceActor.java | 2 +- .../server/datanode/SimulatedFSDataset.java | 2 +- .../hdfs/server/datanode/TestBlockReport.java | 190 ++++++------------ 6 files changed, 86 insertions(+), 163 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index b30e984db52..81a00bc6e74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -54,3 +54,6 @@ IMPROVEMENTS: HDFS-5435. File append fails to initialize storageIDs. (Junping Du via Arpit Agarwal) + + HDFS-5437. Fix TestBlockReport and TestBPOfferService failures. (Arpit + Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java index a6c2df46b50..a7f5ff16f75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java @@ -19,7 +19,9 @@ package org.apache.hadoop.hdfs.protocol; import java.util.Iterator; import java.util.List; +import java.util.Random; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; @@ -250,33 +252,28 @@ public class BlockListAsLongs implements Iterable { } /** - * The block-id of the indexTh block - * @param index - the block whose block-id is desired - * @return the block-id + * Corrupt the generation stamp of the block with the given index. + * Not meant to be used outside of tests. */ - @Deprecated - public long getBlockId(final int index) { - return blockId(index); - } - - /** - * The block-len of the indexTh block - * @param index - the block whose block-len is desired - * @return - the block-len - */ - @Deprecated - public long getBlockLen(final int index) { - return blockLength(index); + @VisibleForTesting + public long corruptBlockGSForTesting(final int blockIndex, Random rand) { + long oldGS = blockList[index2BlockId(blockIndex) + 2]; + while (blockList[index2BlockId(blockIndex) + 2] == oldGS) { + blockList[index2BlockId(blockIndex) + 2] = rand.nextInt(); + } + return oldGS; } /** - * The generation stamp of the indexTh block - * @param index - the block whose block-len is desired - * @return - the generation stamp + * Corrupt the length of the block with the given index by truncation. + * Not meant to be used outside of tests. */ - @Deprecated - public long getBlockGenStamp(final int index) { - return blockGenerationStamp(index); + @VisibleForTesting + public long corruptBlockLengthForTesting(final int blockIndex, Random rand) { + long oldLength = blockList[index2BlockId(blockIndex) + 1]; + blockList[index2BlockId(blockIndex) + 1] = + rand.nextInt((int) oldLength - 1); + return oldLength; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java index 0db9b412623..282151651fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java @@ -60,11 +60,7 @@ public class LocatedBlock { private static final DatanodeInfo[] EMPTY_LOCS = new DatanodeInfo[0]; public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs) { - this(b, locs, -1); // startOffset is unknown - } - - public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, long startOffset) { - this(b, locs, startOffset, false); + this(b, locs, -1, false); // startOffset is unknown } public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, long startOffset, @@ -76,6 +72,11 @@ public class LocatedBlock { this(b, storages, -1, false); // startOffset is unknown } + public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, + String[] storageIDs, StorageType[] storageTypes) { + this(b, locs, storageIDs, storageTypes, -1, false, EMPTY_LOCS); + } + public LocatedBlock(ExtendedBlock b, DatanodeStorageInfo[] storages, long startOffset, boolean corrupt) { this(b, DatanodeStorageInfo.toDatanodeInfos(storages), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 8bb48862583..8eb6527053b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -252,7 +252,7 @@ class BPServiceActor implements Runnable { // TODO: Corrupt flag is set to false for compatibility. We can probably // set it to true here. LocatedBlock[] blocks = { - new LocatedBlock(block, dnArr, uuids, types, -1, false, null) }; + new LocatedBlock(block, dnArr, uuids, types) }; try { bpNamenode.reportBadBlocks(blocks); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 47f5d9ea3d0..07ae075a5ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -1029,7 +1029,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { @Override public StorageReport[] getStorageReports(String bpid) { - throw new UnsupportedOperationException(); + return new StorageReport[0]; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java index 21d0339888a..16c15345e32 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java @@ -116,52 +116,45 @@ public class TestBlockReport { cluster.shutdown(); } - private static StorageBlockReport[] getBlockReports(DataNode dn, String bpid) { + // Generate a block report, optionally corrupting the generation + // stamp and/or length of one block. + private static StorageBlockReport[] getBlockReports( + DataNode dn, String bpid, boolean corruptOneBlockGs, + boolean corruptOneBlockLen) { Map perVolumeBlockLists = dn.getFSDataset().getBlockReports(bpid); // Send block report StorageBlockReport[] reports = new StorageBlockReport[perVolumeBlockLists.size()]; + boolean corruptedGs = false; + boolean corruptedLen = false; - int i = 0; + int reportIndex = 0; for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { String storageID = kvPair.getKey(); - long[] blockList = kvPair.getValue().getBlockListAsLongs(); + BlockListAsLongs blockList = kvPair.getValue(); - // Dummy DatanodeStorage object just for sending the block report. - DatanodeStorage dnStorage = new DatanodeStorage(storageID); - reports[i++] = new StorageBlockReport(dnStorage, blockList); - } - - return reports; - } - - // Get block reports but modify the GS of one of the blocks. - private static StorageBlockReport[] getBlockReportsCorruptSingleBlockGS( - DataNode dn, String bpid) { - Map perVolumeBlockLists = - dn.getFSDataset().getBlockReports(bpid); - - // Send block report - StorageBlockReport[] reports = - new StorageBlockReport[perVolumeBlockLists.size()]; - - boolean corruptedBlock = false; - - int i = 0; - for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { - String storageID = kvPair.getKey(); - long[] blockList = kvPair.getValue().getBlockListAsLongs(); - - if (!corruptedBlock) { - blockList[4] = rand.nextInt(); // Bad GS. - corruptedBlock = true; + // Walk the list of blocks until we find one each to corrupt the + // generation stamp and length, if so requested. + for (int i = 0; i < blockList.getNumberOfBlocks(); ++i) { + if (corruptOneBlockGs && !corruptedGs) { + blockList.corruptBlockGSForTesting(i, rand); + LOG.info("Corrupted the GS for block ID " + i); + corruptedGs = true; + } else if (corruptOneBlockLen && !corruptedLen) { + blockList.corruptBlockLengthForTesting(i, rand); + LOG.info("Corrupted the length for block ID " + i); + corruptedLen = true; + } else { + break; + } } // Dummy DatanodeStorage object just for sending the block report. DatanodeStorage dnStorage = new DatanodeStorage(storageID); - reports[i++] = new StorageBlockReport(dnStorage, blockList); + reports[reportIndex++] = + new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs()); } return reports; @@ -207,7 +200,7 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N0); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] reports = getBlockReports(dn, poolId); + StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false); cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); List blocksAfterReport = @@ -289,7 +282,7 @@ public class TestBlockReport { // all blocks belong to the same file, hence same BP String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn0.getDNRegistrationForBP(poolId); - StorageBlockReport[] reports = getBlockReports(dn0, poolId); + StorageBlockReport[] reports = getBlockReports(dn0, poolId, false, false); cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); BlockManagerTestUtil.getComputedDatanodeWork(cluster.getNamesystem() @@ -315,14 +308,13 @@ public class TestBlockReport { public void blockReport_03() throws IOException { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); - DFSTestUtil.createFile(fs, filePath, - FILE_SIZE, REPL_FACTOR, rand.nextLong()); + ArrayList blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath); // all blocks belong to the same file, hence same BP DataNode dn = cluster.getDataNodes().get(DN_N0); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] reports = getBlockReportsCorruptSingleBlockGS(dn, poolId); + StorageBlockReport[] reports = getBlockReports(dn, poolId, true, false); DatanodeCommand dnCmd = cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); if(LOG.isDebugEnabled()) { @@ -362,7 +354,7 @@ public class TestBlockReport { dn.getFSDataset().createRbw(b); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] reports = getBlockReports(dn, poolId); + StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false); DatanodeCommand dnCmd = cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); if(LOG.isDebugEnabled()) { @@ -376,15 +368,6 @@ public class TestBlockReport { cluster.getNamesystem().getPendingDeletionBlocks(), is(1L)); } - // Client requests new block from NN. The test corrupts this very block - // and forces new block report. - // The test case isn't specific for BlockReport because it relies on - // BlockScanner which is out of scope of this test - // Keeping the name to be in synch with the test plan - // - public void blockReport_05() { - } - /** * Test creates a file and closes it. * The second datanode is started in the cluster. @@ -399,14 +382,14 @@ public class TestBlockReport { Path filePath = new Path("/" + METHOD_NAME + ".dat"); final int DN_N1 = DN_N0 + 1; - ArrayList blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath); + writeFile(METHOD_NAME, FILE_SIZE, filePath); startDNandWait(filePath, true); - // all blocks belong to the same file, hence same BP + // all blocks belong to the same file, hence same BP DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] reports = getBlockReports(dn, poolId); + StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false); cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); printStats(); assertEquals("Wrong number of PendingReplication Blocks", @@ -427,66 +410,40 @@ public class TestBlockReport { * @throws IOException in case of an error */ @Test - // Currently this test is failing as expected 'cause the correct behavior is - // not yet implemented (9/15/09) public void blockReport_07() throws Exception { final String METHOD_NAME = GenericTestUtils.getMethodName(); Path filePath = new Path("/" + METHOD_NAME + ".dat"); final int DN_N1 = DN_N0 + 1; // write file and start second node to be "older" than the original - ArrayList blocks = writeFile(METHOD_NAME, FILE_SIZE, filePath); + writeFile(METHOD_NAME, FILE_SIZE, filePath); startDNandWait(filePath, true); - int randIndex = rand.nextInt(blocks.size()); - // Get a block and screw its GS - Block corruptedBlock = blocks.get(randIndex); - String secondNode = cluster.getDataNodes().get(DN_N1).getDatanodeId().getDatanodeUuid(); - if(LOG.isDebugEnabled()) { - LOG.debug("Working with " + secondNode); - LOG.debug("BlockGS before " + blocks.get(randIndex).getGenerationStamp()); - } - corruptBlockGS(corruptedBlock); - if(LOG.isDebugEnabled()) { - LOG.debug("BlockGS after " + blocks.get(randIndex).getGenerationStamp()); - LOG.debug("Done corrupting GS of " + corruptedBlock.getBlockName()); - } // all blocks belong to the same file, hence same BP DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = getBlockReports(dn, poolId); - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); - printStats(); - assertEquals("Wrong number of Corrupted blocks", - 1, cluster.getNamesystem().getCorruptReplicaBlocks() + -// the following might have to be added into the equation if -// the same block could be in two different states at the same time -// and then the expected number of has to be changed to '2' -// cluster.getNamesystem().getPendingReplicationBlocks() + - cluster.getNamesystem().getPendingDeletionBlocks()); - - // Get another block and screw its length to be less than original - if (randIndex == 0) - randIndex++; - else - randIndex--; - corruptedBlock = blocks.get(randIndex); - corruptBlockLen(corruptedBlock); - if(LOG.isDebugEnabled()) { - LOG.debug("Done corrupting length of " + corruptedBlock.getBlockName()); - } - - report[0] = new StorageBlockReport( - report[0].getStorage(), - new BlockListAsLongs(blocks, null).getBlockListAsLongs()); - cluster.getNameNodeRpc().blockReport(dnR, poolId, report); + StorageBlockReport[] reports = getBlockReports(dn, poolId, true, false); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); printStats(); - assertEquals("Wrong number of Corrupted blocks", - 2, cluster.getNamesystem().getCorruptReplicaBlocks() + - cluster.getNamesystem().getPendingReplicationBlocks() + - cluster.getNamesystem().getPendingDeletionBlocks()); + assertThat("Wrong number of corrupt blocks", + cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L)); + assertThat("Wrong number of PendingDeletion blocks", + cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); + assertThat("Wrong number of PendingReplication blocks", + cluster.getNamesystem().getPendingReplicationBlocks(), is(0L)); + + reports = getBlockReports(dn, poolId, true, true); + cluster.getNameNodeRpc().blockReport(dnR, poolId, reports); + printStats(); + + assertThat("Wrong number of corrupt blocks", + cluster.getNamesystem().getCorruptReplicaBlocks(), is(2L)); + assertThat("Wrong number of PendingDeletion blocks", + cluster.getNamesystem().getPendingDeletionBlocks(), is(0L)); + assertThat("Wrong number of PendingReplication blocks", + cluster.getNamesystem().getPendingReplicationBlocks(), is(0L)); printStats(); @@ -529,7 +486,7 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = getBlockReports(dn, poolId); + StorageBlockReport[] report = getBlockReports(dn, poolId, false, false); cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); assertEquals("Wrong number of PendingReplication blocks", @@ -560,14 +517,11 @@ public class TestBlockReport { // write file and start second node to be "older" than the original try { - ArrayList blocks = - writeFile(METHOD_NAME, 12 * bytesChkSum, filePath); + writeFile(METHOD_NAME, 12 * bytesChkSum, filePath); Block bl = findBlock(filePath, 12 * bytesChkSum); BlockChecker bc = new BlockChecker(filePath); bc.start(); - corruptBlockGS(bl); - corruptBlockLen(bl); waitForTempReplica(bl, DN_N1); @@ -575,7 +529,7 @@ public class TestBlockReport { DataNode dn = cluster.getDataNodes().get(DN_N1); String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId); - StorageBlockReport[] report = getBlockReports(dn, poolId); + StorageBlockReport[] report = getBlockReports(dn, poolId, true, true); cluster.getNameNodeRpc().blockReport(dnR, poolId, report); printStats(); assertEquals("Wrong number of PendingReplication blocks", @@ -851,38 +805,6 @@ public class TestBlockReport { ((Log4JLogger) TestBlockReport.LOG).getLogger().setLevel(Level.ALL); } - private void corruptBlockLen(final Block block) - throws IOException { - if (block == null) { - throw new IOException("Block isn't suppose to be null"); - } - long oldLen = block.getNumBytes(); - long newLen = oldLen - rand.nextLong(); - assertTrue("Old and new length shouldn't be the same", - block.getNumBytes() != newLen); - block.setNumBytes(newLen); - if(LOG.isDebugEnabled()) { - LOG.debug("Length of " + block.getBlockName() + - " is changed to " + newLen + " from " + oldLen); - } - } - - private void corruptBlockGS(final Block block) - throws IOException { - if (block == null) { - throw new IOException("Block isn't suppose to be null"); - } - long oldGS = block.getGenerationStamp(); - long newGS = oldGS - rand.nextLong(); - assertTrue("Old and new GS shouldn't be the same", - block.getGenerationStamp() != newGS); - block.setGenerationStamp(newGS); - if(LOG.isDebugEnabled()) { - LOG.debug("Generation stamp of " + block.getBlockName() + - " is changed to " + block.getGenerationStamp() + " from " + oldGS); - } - } - private Block findBlock(Path path, long size) throws IOException { Block ret; List lbs = From cbdeba29bdbd5a8c7f6c9063b957de2826536bc5 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sat, 2 Nov 2013 05:21:48 +0000 Subject: [PATCH 27/76] HDFS-5447. Fix TestJspHelper in branch HDFS-2832. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1538145 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 +++ .../blockmanagement/DatanodeDescriptor.java | 25 ++----------------- .../hdfs/server/common/TestJspHelper.java | 24 +++++++++++++++--- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 81a00bc6e74..8bf19e6a908 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -57,3 +57,6 @@ IMPROVEMENTS: HDFS-5437. Fix TestBlockReport and TestBPOfferService failures. (Arpit Agarwal) + + HDFS-5447. Fix TestJspHelper. (Arpit Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 81568240bb6..1fe1fc4efe3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -43,9 +43,6 @@ import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.util.IntrusiveCollection; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; - /** * This class extends the DatanodeInfo class with ephemeral information (eg * health, capacity, what blocks are associated with the Datanode) that is @@ -218,26 +215,7 @@ public class DatanodeDescriptor extends DatanodeInfo { */ public DatanodeDescriptor(DatanodeID nodeID, String networkLocation) { - this(nodeID, networkLocation, 0, 0, 0, 0); - } - - /** - * DatanodeDescriptor constructor - * @param nodeID id of the data node - * @param networkLocation location of the data node in network - * @param cacheCapacity cache capacity of the data node - * @param cacheUsed cache used on the data node - * @param xceiverCount # of data transfers at the data node - */ - public DatanodeDescriptor(DatanodeID nodeID, - String networkLocation, - long cacheCapacity, - long cacheUsed, - int xceiverCount, - int failedVolumes) { super(nodeID, networkLocation); - updateHeartbeat(StorageReport.EMPTY_ARRAY, cacheCapacity, cacheUsed, - xceiverCount, failedVolumes); } /** @@ -638,7 +616,8 @@ public class DatanodeDescriptor extends DatanodeInfo { return sb.toString(); } - DatanodeStorageInfo updateStorage(DatanodeStorage s) { + @VisibleForTesting + public DatanodeStorageInfo updateStorage(DatanodeStorage s) { synchronized (storageMap) { DatanodeStorageInfo storage = storageMap.get(s.getStorageID()); if (storage == null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java index ece05e7f266..93b64bf0009 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java @@ -25,6 +25,8 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.web.resources.DoAsParam; import org.apache.hadoop.hdfs.web.resources.UserParam; import org.apache.hadoop.io.DataInputBuffer; @@ -451,10 +453,24 @@ public class TestJspHelper { 1234, 2345, 3456, 4567); DatanodeID dnId2 = new DatanodeID("127.0.0.2", "localhost2", "datanode2", 1235, 2346, 3457, 4568); - DatanodeDescriptor dnDesc1 = new DatanodeDescriptor( - dnId1, "rack1", 5l, 3l, 10, 2); - DatanodeDescriptor dnDesc2 = new DatanodeDescriptor( - dnId2, "rack2", 10l, 2l, 20, 1); + + // Setup DatanodeDescriptors with one storage each. + DatanodeDescriptor dnDesc1 = new DatanodeDescriptor(dnId1, "rack1"); + DatanodeDescriptor dnDesc2 = new DatanodeDescriptor(dnId2, "rack2"); + + // Update the DatanodeDescriptors with their attached storages. + dnDesc1.updateStorage(new DatanodeStorage("dnStorage1")); + dnDesc2.updateStorage(new DatanodeStorage("dnStorage2")); + + StorageReport[] report1 = new StorageReport[] { + new StorageReport("dnStorage1", false, 1024, 100, 924, 100) + }; + StorageReport[] report2 = new StorageReport[] { + new StorageReport("dnStorage2", false, 2500, 200, 1848, 200) + }; + dnDesc1.updateHeartbeat(report1, 5l, 3l, 10, 2); + dnDesc2.updateHeartbeat(report2, 10l, 2l, 20, 1); + ArrayList live = new ArrayList(); live.add(dnDesc1); live.add(dnDesc2); From 26a1fda51e325377734c90399850ff3aa44b5bc1 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Sun, 3 Nov 2013 17:49:08 +0000 Subject: [PATCH 28/76] HDFS-5452. Fix TestReplicationPolicy and TestBlocksScheduledCounter. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1538407 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 + .../server/blockmanagement/BlockInfo.java | 5 +- .../server/blockmanagement/BlockManager.java | 11 +---- .../BlockPlacementPolicyDefault.java | 6 +-- .../blockmanagement/DatanodeDescriptor.java | 46 ++++++++++++++---- .../blockmanagement/DatanodeStorageInfo.java | 47 +------------------ .../TestReplicationPolicy.java | 2 +- 7 files changed, 50 insertions(+), 69 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 8bf19e6a908..20ec6aafdd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -60,3 +60,5 @@ IMPROVEMENTS: HDFS-5447. Fix TestJspHelper. (Arpit Agarwal) + HDFS-5452. Fix TestReplicationPolicy and TestBlocksScheduledCounter. + (szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index e0321ed1fd9..c1d05655e7c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -94,8 +94,7 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { DatanodeStorageInfo getStorageInfo(int index) { assert this.triplets != null : "BlockInfo is not initialized"; assert index >= 0 && index*3 < triplets.length : "Index is out of bound"; - DatanodeStorageInfo storage = (DatanodeStorageInfo)triplets[index*3]; - return storage; + return (DatanodeStorageInfo)triplets[index*3]; } private BlockInfo getPrevious(int index) { @@ -118,7 +117,7 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { return info; } - void setStorageInfo(int index, DatanodeStorageInfo storage) { + private void setStorageInfo(int index, DatanodeStorageInfo storage) { assert this.triplets != null : "BlockInfo is not initialized"; assert index >= 0 && index*3 < triplets.length : "Index is out of bound"; triplets[index*3] = storage; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 441b3a109e4..180c919a0d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2628,17 +2628,10 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block @VisibleForTesting void addBlock(DatanodeDescriptor node, String storageID, Block block, String delHint) throws IOException { - // Decrement number of blocks scheduled to this storage. + // Decrement number of blocks scheduled to this datanode. // for a retry request (of DatanodeProtocol#blockReceivedAndDeleted with // RECEIVED_BLOCK), we currently also decrease the approximate number. - DatanodeStorageInfo storageInfo = node.getStorageInfo(storageID); - if (storageInfo != null) { - storageInfo.decrementBlocksScheduled(); - } else { - throw new IllegalArgumentException( - "Unrecognized storageID " + storageID + " in block report " + - "from Datanode " + node.toString()); - } + node.decrementBlocksScheduled(); // get the deletion hint node DatanodeDescriptor delHintNode = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 99a40e38b3b..8cd5dd6be50 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -620,9 +620,9 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { } final long requiredSize = blockSize * HdfsConstants.MIN_BLOCKS_FOR_WRITE; - final long scheduledSize = blockSize = storage.getBlocksScheduled(); - if (requiredSize > storage.getRemaining() - scheduledSize) { - logNodeIsNotChosen(storage, "the storage does not have enough space "); + final long scheduledSize = blockSize * node.getBlocksScheduled(); + if (requiredSize > node.getRemaining() - scheduledSize) { + logNodeIsNotChosen(storage, "the node does not have enough space "); return false; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 1fe1fc4efe3..3153dd0b172 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -27,22 +27,22 @@ import java.util.List; import java.util.Map; import java.util.Queue; -import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.server.namenode.CachedBlock; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.StorageReport; -import org.apache.hadoop.hdfs.server.namenode.CachedBlock; import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.util.IntrusiveCollection; import org.apache.hadoop.util.Time; +import com.google.common.annotations.VisibleForTesting; + /** * This class extends the DatanodeInfo class with ephemeral information (eg * health, capacity, what blocks are associated with the Datanode) that is @@ -192,6 +192,15 @@ public class DatanodeDescriptor extends DatanodeInfo { /** A set of blocks to be invalidated by this datanode */ private LightWeightHashSet invalidateBlocks = new LightWeightHashSet(); + /* Variables for maintaining number of blocks scheduled to be written to + * this storage. This count is approximate and might be slightly bigger + * in case of errors (e.g. datanode does not report if an error occurs + * while writing the block). + */ + private int currApproxBlocksScheduled = 0; + private int prevApproxBlocksScheduled = 0; + private long lastBlocksScheduledRollTime = 0; + private static final int BLOCKS_SCHEDULED_ROLL_INTERVAL = 600*1000; //10min private int volumeFailures = 0; /** @@ -342,7 +351,7 @@ public class DatanodeDescriptor extends DatanodeInfo { for (StorageReport report : reports) { DatanodeStorageInfo storage = storageMap.get(report.getStorageID()); if (storage != null) { - storage.receivedHeartbeat(report, getLastUpdate()); + storage.receivedHeartbeat(report); totalCapacity += report.getCapacity(); totalRemaining += report.getRemaining(); totalBlockPoolUsed += report.getBlockPoolUsed(); @@ -354,6 +363,7 @@ public class DatanodeDescriptor extends DatanodeInfo { LOG.warn("Unrecognized storage ID " + report.getStorageID()); } } + rollBlocksScheduled(getLastUpdate()); // Update total metrics for the node. setCapacity(totalCapacity); @@ -481,11 +491,31 @@ public class DatanodeDescriptor extends DatanodeInfo { * to this datanode. */ public int getBlocksScheduled() { - int n = 0; - for(DatanodeStorageInfo storage : getStorageInfos()) { - n += storage.getBlocksScheduled(); + return currApproxBlocksScheduled + prevApproxBlocksScheduled; + } + + /** Increment the number of blocks scheduled. */ + void incrementBlocksScheduled() { + currApproxBlocksScheduled++; + } + + /** Decrement the number of blocks scheduled. */ + void decrementBlocksScheduled() { + if (prevApproxBlocksScheduled > 0) { + prevApproxBlocksScheduled--; + } else if (currApproxBlocksScheduled > 0) { + currApproxBlocksScheduled--; + } + // its ok if both counters are zero. + } + + /** Adjusts curr and prev number of blocks scheduled every few minutes. */ + private void rollBlocksScheduled(long now) { + if (now - lastBlocksScheduledRollTime > BLOCKS_SCHEDULED_ROLL_INTERVAL) { + prevApproxBlocksScheduled = currApproxBlocksScheduled; + currApproxBlocksScheduled = 0; + lastBlocksScheduledRollTime = now; } - return n; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index e5f4e8b21d3..440a3cfe818 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -121,16 +121,6 @@ public class DatanodeStorageInfo { */ private boolean blockContentsStale = true; - /* Variables for maintaining number of blocks scheduled to be written to - * this storage. This count is approximate and might be slightly bigger - * in case of errors (e.g. datanode does not report if an error occurs - * while writing the block). - */ - private int currApproxBlocksScheduled = 0; - private int prevApproxBlocksScheduled = 0; - private long lastBlocksScheduledRollTime = 0; - private static final int BLOCKS_SCHEDULED_ROLL_INTERVAL = 600*1000; //10min - public DatanodeStorageInfo(DatanodeDescriptor dn, DatanodeStorage s) { this.dn = dn; this.storageID = s.getStorageID(); @@ -155,10 +145,9 @@ public class DatanodeStorageInfo { blockContentsStale = true; } - void receivedHeartbeat(StorageReport report, final long lastUpdate) { + void receivedHeartbeat(StorageReport report) { updateState(report); heartbeatedSinceFailover = true; - rollBlocksScheduled(lastUpdate); } void receivedBlockReport() { @@ -249,42 +238,10 @@ public class DatanodeStorageInfo { return dn; } - /** - * @return Approximate number of blocks currently scheduled to be written - * to this storage. - */ - int getBlocksScheduled() { - return currApproxBlocksScheduled + prevApproxBlocksScheduled; - } - /** Increment the number of blocks scheduled for each given storage */ public static void incrementBlocksScheduled(DatanodeStorageInfo... storages) { for (DatanodeStorageInfo s : storages) { - s.incrementBlocksScheduled(); - } - } - - /** Increment the number of blocks scheduled. */ - private void incrementBlocksScheduled() { - currApproxBlocksScheduled++; - } - - /** Decrement the number of blocks scheduled. */ - void decrementBlocksScheduled() { - if (prevApproxBlocksScheduled > 0) { - prevApproxBlocksScheduled--; - } else if (currApproxBlocksScheduled > 0) { - currApproxBlocksScheduled--; - } - // its ok if both counters are zero. - } - - /** Adjusts curr and prev number of blocks scheduled every few minutes. */ - private void rollBlocksScheduled(long now) { - if (now - lastBlocksScheduledRollTime > BLOCKS_SCHEDULED_ROLL_INTERVAL) { - prevApproxBlocksScheduled = currApproxBlocksScheduled; - currApproxBlocksScheduled = 0; - lastBlocksScheduledRollTime = now; + s.getDatanodeDescriptor().incrementBlocksScheduled(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index 978009a3117..7bad49f1f83 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -474,7 +474,7 @@ public class TestReplicationPolicy { assertFalse(log.size() == 0); final LoggingEvent lastLogEntry = log.get(log.size() - 1); - assertEquals(lastLogEntry.getLevel(), Level.WARN); + assertTrue(Level.WARN.isGreaterOrEqual(lastLogEntry.getLevel())); // Suppose to place replicas on each node but two data nodes are not // available for placing replica, so here we expect a short of 2 assertTrue(((String)lastLogEntry.getMessage()).contains("in need of 2")); From 9660bfa84c900afd4824feb62d14256584edfb95 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 4 Nov 2013 02:13:04 +0000 Subject: [PATCH 29/76] HDFS-5448. Datanode should generate its ID on first registration git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1538496 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 5 +++- .../hadoop/hdfs/protocol/DatanodeID.java | 5 ---- .../blockmanagement/DatanodeManager.java | 12 -------- .../hadoop/hdfs/server/datanode/DataNode.java | 29 ++++++++++++++----- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 20ec6aafdd2..74483081067 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -61,4 +61,7 @@ IMPROVEMENTS: HDFS-5447. Fix TestJspHelper. (Arpit Agarwal) HDFS-5452. Fix TestReplicationPolicy and TestBlocksScheduledCounter. - (szetszwo) + + HDFS-5448. Datanode should generate its ID on first registration. (Arpit + Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java index b8b84f56662..a00fdf47fab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java @@ -113,11 +113,6 @@ public class DatanodeID implements Comparable { } } - public String generateNewDatanodeUuid() { - datanodeUuid = UUID.randomUUID().toString(); - return datanodeUuid; - } - /** * @return ipAddr; */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 892caa0c596..ec943a5a094 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -845,18 +845,6 @@ public class DatanodeManager { return; } - // This is a new datanode. - if (nodeReg.getDatanodeUuid() == null || - nodeReg.getDatanodeUuid().isEmpty()) { - // this data node has never been registered - nodeReg.generateNewDatanodeUuid(); - if (NameNode.stateChangeLog.isDebugEnabled()) { - NameNode.stateChangeLog.debug( - "BLOCK* NameSystem.registerDatanode: " - + "new Datanode UUID " + nodeReg.getDatanodeUuid() + " assigned."); - } - } - DatanodeDescriptor nodeDescr = new DatanodeDescriptor(nodeReg, NetworkTopology.DEFAULT_RACK); boolean success = false; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index a493423695a..fbccecae67f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -693,16 +693,35 @@ public class DataNode extends Configured readaheadPool = ReadaheadPool.getInstance(); } + /** + * Verify that the DatanodeUuid has been initialized. If this is a new + * datanode then we generate a new Datanode Uuid and persist it to disk. + * + * @throws IOException + */ + private synchronized void checkDatanodeUuid() throws IOException { + if (storage.getDatanodeUuid() == null) { + storage.setDatanodeUuid(UUID.randomUUID().toString()); + storage.writeAll(); + LOG.info("Generated and persisted new Datanode UUID " + + storage.getDatanodeUuid()); + } + } + /** * Create a DatanodeRegistration for a specific block pool. * @param nsInfo the namespace info from the first part of the NN handshake */ - DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) { + DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) + throws IOException { StorageInfo storageInfo = storage.getBPStorage(nsInfo.getBlockPoolID()); if (storageInfo == null) { // it's null in the case of SimulatedDataSet storageInfo = new StorageInfo(nsInfo); } + + checkDatanodeUuid(); + DatanodeID dnId = new DatanodeID( streamingAddr.getAddress().getHostAddress(), hostName, storage.getDatanodeUuid(), getXferPort(), getInfoPort(), @@ -724,13 +743,7 @@ public class DataNode extends Configured id = bpRegistration; } - if (storage.getDatanodeUuid() == null) { - // This is a fresh datanode, persist the NN-provided Datanode ID - storage.setDatanodeUuid(bpRegistration.getDatanodeUuid()); - storage.writeAll(); - LOG.info("Datanode ID " + bpRegistration.getDatanodeUuid() - + " is assigned to new storage " + bpRegistration); - } else if(!storage.getDatanodeUuid().equals(bpRegistration.getDatanodeUuid())) { + if(!storage.getDatanodeUuid().equals(bpRegistration.getDatanodeUuid())) { throw new IOException("Inconsistent Datanode IDs. Name-node returned " + bpRegistration.getDatanodeUuid() + ". Expecting " + storage.getDatanodeUuid()); From 0091606e2d802e7c59e29745963c50ec0ca54aac Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 4 Nov 2013 16:00:12 +0000 Subject: [PATCH 30/76] HDFS-5448. Fix break caused by previous checkin for HDFS-5448. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1538655 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 +++ .../org/apache/hadoop/hdfs/server/datanode/BPOfferService.java | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 74483081067..b80fff3fc54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -65,3 +65,6 @@ IMPROVEMENTS: HDFS-5448. Datanode should generate its ID on first registration. (Arpit Agarwal) + HDFS-5448. Fix break caused by previous checkin for HDFS-5448. (Arpit + Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index 084b9de9120..5caf54fc6b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -331,7 +331,8 @@ class BPOfferService { } } - synchronized DatanodeRegistration createRegistration() { + synchronized DatanodeRegistration createRegistration() + throws IOException { Preconditions.checkState(bpNSInfo != null, "getRegistration() can only be called after initial handshake"); return dn.createBPRegistration(bpNSInfo); From 1d357365b7f01ec783a8b2a10d270ad33267018e Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 4 Nov 2013 21:47:51 +0000 Subject: [PATCH 31/76] HDFS-5455. NN should update storageMap on first heartbeat. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1538787 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 ++ .../blockmanagement/DatanodeDescriptor.java | 22 +++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index b80fff3fc54..ffeb3155e06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -68,3 +68,5 @@ IMPROVEMENTS: HDFS-5448. Fix break caused by previous checkin for HDFS-5448. (Arpit Agarwal) + HDFS-5455. NN should update storageMap on first heartbeat. (Arpit Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 3153dd0b172..ad499912c2e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -350,18 +350,16 @@ public class DatanodeDescriptor extends DatanodeInfo { this.volumeFailures = volFailures; for (StorageReport report : reports) { DatanodeStorageInfo storage = storageMap.get(report.getStorageID()); - if (storage != null) { - storage.receivedHeartbeat(report); - totalCapacity += report.getCapacity(); - totalRemaining += report.getRemaining(); - totalBlockPoolUsed += report.getBlockPoolUsed(); - totalDfsUsed += report.getDfsUsed(); - } else { - // This warning is generally benign during cluster initialization - // when the heartbeat is received before the initial block reports - // from each storage. - LOG.warn("Unrecognized storage ID " + report.getStorageID()); + if (storage == null) { + // This is seen during cluster initialization when the heartbeat + // is received before the initial block reports from each storage. + storage = updateStorage(new DatanodeStorage(report.getStorageID())); } + storage.receivedHeartbeat(report); + totalCapacity += report.getCapacity(); + totalRemaining += report.getRemaining(); + totalBlockPoolUsed += report.getBlockPoolUsed(); + totalDfsUsed += report.getDfsUsed(); } rollBlocksScheduled(getLastUpdate()); @@ -651,6 +649,8 @@ public class DatanodeDescriptor extends DatanodeInfo { synchronized (storageMap) { DatanodeStorageInfo storage = storageMap.get(s.getStorageID()); if (storage == null) { + LOG.info("Adding new storage ID " + s.getStorageID() + + " for DN " + getXferAddr()); storage = new DatanodeStorageInfo(this, s); storageMap.put(s.getStorageID(), storage); } else { From d2b7b6589d30f50f86dbef25aeb322f817f037d8 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 4 Nov 2013 22:49:00 +0000 Subject: [PATCH 32/76] HDFS-5457. Fix TestDatanodeRegistration, TestFsck and TestAddBlockRetry. (Contributed by szetszwo) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1538794 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 +++ .../hdfs/server/blockmanagement/DatanodeDescriptor.java | 2 +- .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 3 ++- .../hadoop/hdfs/server/namenode/TestAddBlockRetry.java | 9 +++++---- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index ffeb3155e06..968b33ab8ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -70,3 +70,6 @@ IMPROVEMENTS: HDFS-5455. NN should update storageMap on first heartbeat. (Arpit Agarwal) + HDFS-5457. Fix TestDatanodeRegistration, TestFsck and TestAddBlockRetry. + (Contributed bt szetszwo) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index ad499912c2e..3fe6af0127e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -385,7 +385,7 @@ public class DatanodeDescriptor extends DatanodeInfo { @Override public boolean hasNext() { update(); - return iterators.get(index).hasNext(); + return !iterators.isEmpty() && iterators.get(index).hasNext(); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index df059c4d167..81991b47b15 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -5774,7 +5774,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, DatanodeInfo[] nodes = blocks[i].getLocations(); String[] storageIDs = blocks[i].getStorageIDs(); for (int j = 0; j < nodes.length; j++) { - blockManager.findAndMarkBlockAsCorrupt(blk, nodes[j], storageIDs[j], + blockManager.findAndMarkBlockAsCorrupt(blk, nodes[j], + storageIDs == null ? null: storageIDs[j], "client machine reported it"); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java index 3114c824cea..08c44c23edb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.net.Node; @@ -99,13 +100,13 @@ public class TestAddBlockRetry { bmField.setAccessible(true); bmField.set(ns, spyBM); - doAnswer(new Answer() { + doAnswer(new Answer() { @Override - public DatanodeDescriptor[] answer(InvocationOnMock invocation) + public DatanodeStorageInfo[] answer(InvocationOnMock invocation) throws Throwable { LOG.info("chooseTarget for " + src); - DatanodeDescriptor[] ret = - (DatanodeDescriptor[]) invocation.callRealMethod(); + DatanodeStorageInfo[] ret = + (DatanodeStorageInfo[]) invocation.callRealMethod(); count++; if(count == 1) { // run second addBlock() LOG.info("Starting second addBlock for " + src); From 3e3fea7cd433a77ab2b92a5035ffc4bdea02c6cf Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 6 Nov 2013 00:25:10 +0000 Subject: [PATCH 33/76] HDFS-5466. Update storage IDs when the pipeline is updated. (Contributed by szetszwo) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1539203 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 5 +++- .../apache/hadoop/hdfs/DFSOutputStream.java | 30 ++++++++++++++----- .../server/blockmanagement/BlockManager.java | 1 - 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 968b33ab8ec..f5ede31804c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -71,5 +71,8 @@ IMPROVEMENTS: HDFS-5455. NN should update storageMap on first heartbeat. (Arpit Agarwal) HDFS-5457. Fix TestDatanodeRegistration, TestFsck and TestAddBlockRetry. - (Contributed bt szetszwo) + (Contributed by szetszwo) + + HDFS-5466. Update storage IDs when the pipeline is updated. (Contributed + by szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 9e91abad6e7..7f5b6ed5323 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -402,8 +402,7 @@ public class DFSOutputStream extends FSOutputSummer } // setup pipeline to append to the last block XXX retries?? - nodes = lastBlock.getLocations(); - storageIDs = lastBlock.getStorageIDs(); + setPipeline(lastBlock); errorIndex = -1; // no errors yet. if (nodes.length < 1) { throw new IOException("Unable to retrieve blocks locations " + @@ -412,6 +411,14 @@ public class DFSOutputStream extends FSOutputSummer } } + + private void setPipeline(LocatedBlock lb) { + setPipeline(lb.getLocations(), lb.getStorageIDs()); + } + private void setPipeline(DatanodeInfo[] nodes, String[] storageIDs) { + this.nodes = nodes; + this.storageIDs = storageIDs; + } private void setFavoredNodes(String[] favoredNodes) { this.favoredNodes = favoredNodes; @@ -435,7 +442,7 @@ public class DFSOutputStream extends FSOutputSummer this.setName("DataStreamer for file " + src); closeResponder(); closeStream(); - nodes = null; + setPipeline(null, null); stage = BlockConstructionStage.PIPELINE_SETUP_CREATE; } @@ -504,7 +511,7 @@ public class DFSOutputStream extends FSOutputSummer if(DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("Allocating new block"); } - nodes = nextBlockOutputStream(); + setPipeline(nextBlockOutputStream()); initDataStreaming(); } else if (stage == BlockConstructionStage.PIPELINE_SETUP_APPEND) { if(DFSClient.LOG.isDebugEnabled()) { @@ -912,7 +919,7 @@ public class DFSOutputStream extends FSOutputSummer src, block, nodes, storageIDs, failed.toArray(new DatanodeInfo[failed.size()]), 1, dfsClient.clientName); - nodes = lb.getLocations(); + setPipeline(lb); //find the new datanode final int d = findNewDatanode(original); @@ -1012,7 +1019,14 @@ public class DFSOutputStream extends FSOutputSummer System.arraycopy(nodes, 0, newnodes, 0, errorIndex); System.arraycopy(nodes, errorIndex+1, newnodes, errorIndex, newnodes.length-errorIndex); - nodes = newnodes; + + final String[] newStorageIDs = new String[newnodes.length]; + System.arraycopy(storageIDs, 0, newStorageIDs, 0, errorIndex); + System.arraycopy(storageIDs, errorIndex+1, newStorageIDs, errorIndex, + newStorageIDs.length-errorIndex); + + setPipeline(newnodes, newStorageIDs); + hasError = false; lastException.set(null); errorIndex = -1; @@ -1051,7 +1065,7 @@ public class DFSOutputStream extends FSOutputSummer * Must get block ID and the IDs of the destinations from the namenode. * Returns the list of target datanodes. */ - private DatanodeInfo[] nextBlockOutputStream() throws IOException { + private LocatedBlock nextBlockOutputStream() throws IOException { LocatedBlock lb = null; DatanodeInfo[] nodes = null; int count = dfsClient.getConf().nBlockWriteRetry; @@ -1093,7 +1107,7 @@ public class DFSOutputStream extends FSOutputSummer if (!success) { throw new IOException("Unable to create new block."); } - return nodes; + return lb; } // connects to the first datanode in the pipeline diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 180c919a0d6..867f4e1802d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1620,7 +1620,6 @@ public class BlockManager { // To minimize startup time, we discard any second (or later) block reports // that we receive while still in startup phase. final DatanodeStorageInfo storageInfo = node.updateStorage(storage); - LOG.info("XXX storageInfo=" + storageInfo + ", storage=" + storage); if (namesystem.isInStartupSafeMode() && storageInfo.getBlockReportCount() > 0) { blockLog.info("BLOCK* processReport: " From fa5ba6d977520f1faaa97c55a50a22c98b3ee109 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 6 Nov 2013 06:50:33 +0000 Subject: [PATCH 34/76] HDFS-5439. Fix TestPendingReplication. (Contributed by Junping Du, Arpit Agarwal) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1539247 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 +++ .../server/blockmanagement/BlockManager.java | 7 +++--- .../BlockPlacementPolicyDefault.java | 14 +++++++----- .../blockmanagement/DatanodeStorageInfo.java | 9 ++++++++ .../PendingReplicationBlocks.java | 22 +++++++++---------- .../hdfs/server/datanode/BPServiceActor.java | 2 -- .../hadoop/hdfs/server/datanode/DataNode.java | 2 +- .../server/protocol/DatanodeRegistration.java | 2 +- .../TestPendingReplication.java | 21 ++++++++++-------- 9 files changed, 50 insertions(+), 32 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index f5ede31804c..1bcd677af6c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -76,3 +76,6 @@ IMPROVEMENTS: HDFS-5466. Update storage IDs when the pipeline is updated. (Contributed by szetszwo) + HDFS-5439. Fix TestPendingReplication. (Contributed by Junping Du, Arpit + Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 867f4e1802d..54b24d1a222 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1332,7 +1332,8 @@ public class BlockManager { // Move the block-replication into a "pending" state. // The reason we use 'pending' is so we can retry // replications that fail after an appropriate amount of time. - pendingReplications.increment(block, targets); + pendingReplications.increment(block, + DatanodeStorageInfo.toDatanodeDescriptors(targets)); if(blockLog.isDebugEnabled()) { blockLog.debug( "BLOCK* block " + block @@ -1357,7 +1358,7 @@ public class BlockManager { StringBuilder targetList = new StringBuilder("datanode(s)"); for (int k = 0; k < targets.length; k++) { targetList.append(' '); - targetList.append(targets[k]); + targetList.append(targets[k].getDatanodeDescriptor()); } blockLog.info("BLOCK* ask " + rw.srcNode + " to replicate " + rw.block + " to " + targetList); @@ -2645,7 +2646,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block // // Modify the blocks->datanode map and node's map. // - pendingReplications.decrement(block, node, storageID); + pendingReplications.decrement(block, node); processAndHandleReportedBlock(node, storageID, block, ReplicaState.FINALIZED, delHintNode); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 8cd5dd6be50..fa1bfe645dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -496,7 +496,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { builder.setLength(0); builder.append("["); } - boolean goodTarget = false; + boolean badTarget = false; DatanodeStorageInfo firstChosen = null; while(numOfReplicas > 0 && numOfAvailableNodes > 0) { DatanodeDescriptor chosenNode = @@ -506,26 +506,30 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { final DatanodeStorageInfo[] storages = DFSUtil.shuffle( chosenNode.getStorageInfos()); - for(int i = 0; i < storages.length && !goodTarget; i++) { + int i; + for(i = 0; i < storages.length; i++) { final int newExcludedNodes = addIfIsGoodTarget(storages[i], excludedNodes, blocksize, maxNodesPerRack, considerLoad, results, avoidStaleNodes, storageType); - goodTarget = newExcludedNodes >= 0; - if (goodTarget) { + if (newExcludedNodes >= 0) { numOfReplicas--; if (firstChosen == null) { firstChosen = storages[i]; } numOfAvailableNodes -= newExcludedNodes; + break; } } + + // If no candidate storage was found on this DN then set badTarget. + badTarget = (i == storages.length); } } if (numOfReplicas>0) { String detail = enableDebugLogging; if (LOG.isDebugEnabled()) { - if (!goodTarget && builder != null) { + if (badTarget && builder != null) { detail = builder.append("]").toString(); builder.setLength(0); } else detail = ""; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index 440a3cfe818..7286755459e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -46,6 +46,15 @@ public class DatanodeStorageInfo { return datanodes; } + static DatanodeDescriptor[] toDatanodeDescriptors( + DatanodeStorageInfo[] storages) { + DatanodeDescriptor[] datanodes = new DatanodeDescriptor[storages.length]; + for (int i = 0; i < storages.length; ++i) { + datanodes[i] = storages[i].getDatanodeDescriptor(); + } + return datanodes; + } + public static String[] toStorageIDs(DatanodeStorageInfo[] storages) { String[] storageIDs = new String[storages.length]; for(int i = 0; i < storageIDs.length; i++) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java index c99d53fc148..9a05c711fcf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java @@ -76,7 +76,7 @@ class PendingReplicationBlocks { * @param block The corresponding block * @param targets The DataNodes where replicas of the block should be placed */ - void increment(Block block, DatanodeStorageInfo[] targets) { + void increment(Block block, DatanodeDescriptor[] targets) { synchronized (pendingReplications) { PendingBlockInfo found = pendingReplications.get(block); if (found == null) { @@ -95,14 +95,14 @@ class PendingReplicationBlocks { * * @param The DataNode that finishes the replication */ - void decrement(Block block, DatanodeDescriptor dn, String storageID) { + void decrement(Block block, DatanodeDescriptor dn) { synchronized (pendingReplications) { PendingBlockInfo found = pendingReplications.get(block); if (found != null) { if(LOG.isDebugEnabled()) { LOG.debug("Removing pending replication for " + block); } - found.decrementReplicas(dn.getStorageInfo(storageID)); + found.decrementReplicas(dn); if (found.getNumReplicas() <= 0) { pendingReplications.remove(block); } @@ -174,12 +174,12 @@ class PendingReplicationBlocks { */ static class PendingBlockInfo { private long timeStamp; - private final List targets; + private final List targets; - PendingBlockInfo(DatanodeStorageInfo[] targets) { + PendingBlockInfo(DatanodeDescriptor[] targets) { this.timeStamp = now(); - this.targets = targets == null ? new ArrayList() - : new ArrayList(Arrays.asList(targets)); + this.targets = targets == null ? new ArrayList() + : new ArrayList(Arrays.asList(targets)); } long getTimeStamp() { @@ -190,16 +190,16 @@ class PendingReplicationBlocks { timeStamp = now(); } - void incrementReplicas(DatanodeStorageInfo... newTargets) { + void incrementReplicas(DatanodeDescriptor... newTargets) { if (newTargets != null) { - for (DatanodeStorageInfo dn : newTargets) { + for (DatanodeDescriptor dn : newTargets) { targets.add(dn); } } } - void decrementReplicas(DatanodeStorageInfo storage) { - targets.remove(storage); + void decrementReplicas(DatanodeDescriptor target) { + targets.remove(target); } int getNumReplicas() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 8eb6527053b..652c990f43c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -267,8 +267,6 @@ class BPServiceActor implements Runnable { /** * Report received blocks and delete hints to the Namenode - * TODO: Fix reportReceivedDeletedBlocks to send reports per-volume. - * * @throws IOException */ private void reportReceivedDeletedBlocks() throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 22778c3cc6b..e2615f4ca50 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -867,7 +867,7 @@ public class DataNode extends Configured final StorageInfo bpStorage = storage.getBPStorage(bpid); LOG.info("Setting up storage: nsid=" + bpStorage.getNamespaceID() + ";bpid=" + bpid + ";lv=" + storage.getLayoutVersion() - + ";nsInfo=" + nsInfo); + + ";nsInfo=" + nsInfo + ";dnuuid=" + storage.getDatanodeUuid()); } synchronized(this) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java index c4c30f8bb8e..8ab18f4b3cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java @@ -82,7 +82,7 @@ public class DatanodeRegistration extends DatanodeID public String toString() { return getClass().getSimpleName() + "(" + getIpAddr() - + ", storageID=" + getDatanodeUuid() + + ", datanodeUuid=" + getDatanodeUuid() + ", infoPort=" + getInfoPort() + ", ipcPort=" + getIpcPort() + ", storageInfo=" + storageInfo diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java index 71da2b3c22d..c63badc9eb7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java @@ -67,7 +67,8 @@ public class TestPendingReplication { Block block = new Block(i, i, 0); DatanodeStorageInfo[] targets = new DatanodeStorageInfo[i]; System.arraycopy(storages, 0, targets, 0, i); - pendingReplications.increment(block, targets); + pendingReplications.increment(block, + DatanodeStorageInfo.toDatanodeDescriptors(targets)); } assertEquals("Size of pendingReplications ", 10, pendingReplications.size()); @@ -77,18 +78,18 @@ public class TestPendingReplication { // remove one item and reinsert it // Block blk = new Block(8, 8, 0); - pendingReplications.decrement(blk, storages[7].getDatanodeDescriptor(), - storages[7].getStorageID()); // removes one replica + pendingReplications.decrement(blk, storages[7].getDatanodeDescriptor()); // removes one replica assertEquals("pendingReplications.getNumReplicas ", 7, pendingReplications.getNumReplicas(blk)); for (int i = 0; i < 7; i++) { // removes all replicas - pendingReplications.decrement(blk, storages[i].getDatanodeDescriptor(), - storages[i].getStorageID()); + pendingReplications.decrement(blk, storages[i].getDatanodeDescriptor()); } assertTrue(pendingReplications.size() == 9); - pendingReplications.increment(blk, DFSTestUtil.createDatanodeStorageInfos(8)); + pendingReplications.increment(blk, + DatanodeStorageInfo.toDatanodeDescriptors( + DFSTestUtil.createDatanodeStorageInfos(8))); assertTrue(pendingReplications.size() == 10); // @@ -116,7 +117,9 @@ public class TestPendingReplication { for (int i = 10; i < 15; i++) { Block block = new Block(i, i, 0); - pendingReplications.increment(block, DFSTestUtil.createDatanodeStorageInfos(i)); + pendingReplications.increment(block, + DatanodeStorageInfo.toDatanodeDescriptors( + DFSTestUtil.createDatanodeStorageInfos(i))); } assertTrue(pendingReplications.size() == 15); @@ -198,7 +201,7 @@ public class TestPendingReplication { DatanodeRegistration dnR = datanodes.get(i).getDNRegistrationForBP( poolId); StorageReceivedDeletedBlocks[] report = { - new StorageReceivedDeletedBlocks(dnR.getDatanodeUuid(), + new StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo( blocks[0], BlockStatus.RECEIVED_BLOCK, "") }) }; cluster.getNameNodeRpc().blockReceivedAndDeleted(dnR, poolId, report); @@ -215,7 +218,7 @@ public class TestPendingReplication { DatanodeRegistration dnR = datanodes.get(i).getDNRegistrationForBP( poolId); StorageReceivedDeletedBlocks[] report = - { new StorageReceivedDeletedBlocks(dnR.getDatanodeUuid(), + { new StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored", new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo( blocks[0], BlockStatus.RECEIVED_BLOCK, "") }) }; cluster.getNameNodeRpc().blockReceivedAndDeleted(dnR, poolId, report); From 75777f1626449e9f45e6e4b1daee644c955bbabc Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 7 Nov 2013 00:13:05 +0000 Subject: [PATCH 35/76] HDFS-5470. Add back trunk's reportDiff algorithm to the branch. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1539504 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 + .../server/blockmanagement/BlockInfo.java | 21 +++ .../server/blockmanagement/BlockManager.java | 26 ++-- .../blockmanagement/DatanodeStorageInfo.java | 29 +++- .../server/blockmanagement/TestBlockInfo.java | 124 ++++++++++++++++++ 5 files changed, 188 insertions(+), 14 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 1bcd677af6c..ad24b08504f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -79,3 +79,5 @@ IMPROVEMENTS: HDFS-5439. Fix TestPendingReplication. (Contributed by Junping Du, Arpit Agarwal) + HDFS-5470. Add back trunk's reportDiff algorithm to the branch. + (szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index c1d05655e7c..c3fb37e2d5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -323,6 +323,27 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { return head; } + /** + * Remove this block from the list of blocks related to the specified + * DatanodeDescriptor. Insert it into the head of the list of blocks. + * + * @return the new head of the list. + */ + public BlockInfo moveBlockToHead(BlockInfo head, DatanodeStorageInfo storage, + int curIndex, int headIndex) { + if (head == this) { + return this; + } + BlockInfo next = this.setNext(curIndex, head); + BlockInfo prev = this.setPrevious(curIndex, null); + + head.setPrevious(headIndex, this); + prev.setNext(prev.findStorageInfo(storage), next); + if (next != null) + next.setPrevious(next.findStorageInfo(storage), prev); + return this; + } + /** * BlockInfo represents a block that is not being constructed. * In order to start modifying the block, the BlockInfo should be converted diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 54b24d1a222..b1ce880441d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1816,13 +1816,15 @@ public class BlockManager { Collection toCorrupt, // add to corrupt replicas list Collection toUC) { // add to under-construction list - dn.updateStorage(storage); + final DatanodeStorageInfo storageInfo = dn.updateStorage(storage); - // add all blocks to remove list - for(Iterator it = dn.getBlockIterator(storage.getStorageID()); - it.hasNext(); ) { - toRemove.add(it.next()); - } + // place a delimiter in the list which separates blocks + // that have been reported from those that have not + BlockInfo delimiter = new BlockInfo(new Block(), 1); + boolean added = storageInfo.addBlock(delimiter); + assert added : "Delimiting block cannot be present in the node"; + int headIndex = 0; //currently the delimiter is in the head of the list + int curIndex; if (newReport == null) newReport = new BlockListAsLongs(); @@ -1834,10 +1836,18 @@ public class BlockManager { BlockInfo storedBlock = processReportedBlock(dn, storage.getStorageID(), iblk, iState, toAdd, toInvalidate, toCorrupt, toUC); - if (storedBlock != null) { - toRemove.remove(storedBlock); + // move block to the head of the list + if (storedBlock != null && (curIndex = storedBlock.findDatanode(dn)) >= 0) { + headIndex = storageInfo.moveBlockToHead(storedBlock, curIndex, headIndex); } } + + // collect blocks that have not been reported + // all of them are next to the delimiter + Iterator it = storageInfo.new BlockIterator(delimiter.getNext(0)); + while(it.hasNext()) + toRemove.add(it.next()); + storageInfo.removeBlock(delimiter); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index 7286755459e..363e4de4d24 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -74,13 +74,11 @@ public class DatanodeStorageInfo { /** * Iterates over the list of blocks belonging to the data-node. */ - static class BlockIterator implements Iterator { + class BlockIterator implements Iterator { private BlockInfo current; - private DatanodeStorageInfo node; - BlockIterator(BlockInfo head, DatanodeStorageInfo dn) { + BlockIterator(BlockInfo head) { this.current = head; - this.node = dn; } public boolean hasNext() { @@ -89,7 +87,7 @@ public class DatanodeStorageInfo { public BlockInfo next() { BlockInfo res = current; - current = current.getNext(current.findStorageInfo(node)); + current = current.getNext(current.findStorageInfo(DatanodeStorageInfo.this)); return res; } @@ -233,7 +231,26 @@ public class DatanodeStorageInfo { } Iterator getBlockIterator() { - return new BlockIterator(this.blockList, this); + return new BlockIterator(blockList); + + } + + /** + * Move block to the head of the list of blocks belonging to the data-node. + * @return the index of the head of the blockList + */ + int moveBlockToHead(BlockInfo b, int curIndex, int headIndex) { + blockList = b.moveBlockToHead(blockList, this, curIndex, headIndex); + return curIndex; + } + + /** + * Used for testing only + * @return the head of the blockList + */ + @VisibleForTesting + protected BlockInfo getHead(){ + return blockList; } public void updateState(StorageReport r) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java new file mode 100644 index 00000000000..79e0c8573d4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.blockmanagement; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.common.GenerationStamp; +import org.junit.Test; + +/** + * This class provides tests for BlockInfo class, which is used in BlocksMap. + * The test covers BlockList.listMoveToHead, used for faster block report + * processing in DatanodeDescriptor.reportDiff. + */ + +public class TestBlockInfo { + + private static final Log LOG = LogFactory + .getLog("org.apache.hadoop.hdfs.TestBlockInfo"); + + @Test + public void testBlockListMoveToHead() throws Exception { + LOG.info("BlockInfo moveToHead tests..."); + + final int MAX_BLOCKS = 10; + + DatanodeStorageInfo dd = DFSTestUtil.createDatanodeStorageInfo("s1", "1.1.1.1"); + ArrayList blockList = new ArrayList(MAX_BLOCKS); + ArrayList blockInfoList = new ArrayList(); + int headIndex; + int curIndex; + + LOG.info("Building block list..."); + for (int i = 0; i < MAX_BLOCKS; i++) { + blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP)); + blockInfoList.add(new BlockInfo(blockList.get(i), 3)); + dd.addBlock(blockInfoList.get(i)); + + // index of the datanode should be 0 + assertEquals("Find datanode should be 0", 0, blockInfoList.get(i) + .findStorageInfo(dd)); + } + + // list length should be equal to the number of blocks we inserted + LOG.info("Checking list length..."); + assertEquals("Length should be MAX_BLOCK", MAX_BLOCKS, dd.numBlocks()); + Iterator it = dd.getBlockIterator(); + int len = 0; + while (it.hasNext()) { + it.next(); + len++; + } + assertEquals("There should be MAX_BLOCK blockInfo's", MAX_BLOCKS, len); + + headIndex = dd.getHead().findStorageInfo(dd); + + LOG.info("Moving each block to the head of the list..."); + for (int i = 0; i < MAX_BLOCKS; i++) { + curIndex = blockInfoList.get(i).findStorageInfo(dd); + headIndex = dd.moveBlockToHead(blockInfoList.get(i), curIndex, headIndex); + // the moved element must be at the head of the list + assertEquals("Block should be at the head of the list now.", + blockInfoList.get(i), dd.getHead()); + } + + // move head of the list to the head - this should not change the list + LOG.info("Moving head to the head..."); + + BlockInfo temp = dd.getHead(); + curIndex = 0; + headIndex = 0; + dd.moveBlockToHead(temp, curIndex, headIndex); + assertEquals( + "Moving head to the head of the list shopuld not change the list", + temp, dd.getHead()); + + // check all elements of the list against the original blockInfoList + LOG.info("Checking elements of the list..."); + temp = dd.getHead(); + assertNotNull("Head should not be null", temp); + int c = MAX_BLOCKS - 1; + while (temp != null) { + assertEquals("Expected element is not on the list", + blockInfoList.get(c--), temp); + temp = temp.getNext(0); + } + + LOG.info("Moving random blocks to the head of the list..."); + headIndex = dd.getHead().findStorageInfo(dd); + Random rand = new Random(); + for (int i = 0; i < MAX_BLOCKS; i++) { + int j = rand.nextInt(MAX_BLOCKS); + curIndex = blockInfoList.get(j).findStorageInfo(dd); + headIndex = dd.moveBlockToHead(blockInfoList.get(j), curIndex, headIndex); + // the moved element must be at the head of the list + assertEquals("Block should be at the head of the list now.", + blockInfoList.get(j), dd.getHead()); + } + } +} \ No newline at end of file From a604e3b73bad8b699fe8952aa018bb115312e97e Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 7 Nov 2013 17:57:30 +0000 Subject: [PATCH 36/76] HDFS-5472. Fix TestDatanodeManager, TestSafeMode and TestNNThroughputBenchmark. (Contributed by szetszwo) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1539736 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 5 ++- .../hadoop/hdfs/protocol/DatanodeID.java | 2 - .../blockmanagement/DatanodeDescriptor.java | 2 + .../hadoop/hdfs/server/datanode/DataNode.java | 6 ++- .../namenode/NNThroughputBenchmark.java | 45 +++++++++++++------ 5 files changed, 43 insertions(+), 17 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index ad24b08504f..ab4303e7ece 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -80,4 +80,7 @@ IMPROVEMENTS: Agarwal) HDFS-5470. Add back trunk's reportDiff algorithm to the branch. - (szetszwo) + (Contributed by szetszwo) + + HDFS-5472. Fix TestDatanodeManager, TestSafeMode and + TestNNThroughputBenchmark (Contributed by szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java index a00fdf47fab..deaf694e182 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java @@ -21,8 +21,6 @@ package org.apache.hadoop.hdfs.protocol; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import java.util.UUID; - /** * This class represents the primary identifier for a Datanode. * Datanodes are identified by how they can be contacted (hostname diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 3fe6af0127e..d0aef08cb4b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -215,6 +215,7 @@ public class DatanodeDescriptor extends DatanodeInfo { */ public DatanodeDescriptor(DatanodeID nodeID) { super(nodeID); + updateHeartbeat(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0); } /** @@ -225,6 +226,7 @@ public class DatanodeDescriptor extends DatanodeInfo { public DatanodeDescriptor(DatanodeID nodeID, String networkLocation) { super(nodeID, networkLocation); + updateHeartbeat(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index e2615f4ca50..2dda60829c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -693,6 +693,10 @@ public class DataNode extends Configured readaheadPool = ReadaheadPool.getInstance(); } + public static String generateUuid() { + return UUID.randomUUID().toString(); + } + /** * Verify that the DatanodeUuid has been initialized. If this is a new * datanode then we generate a new Datanode Uuid and persist it to disk. @@ -701,7 +705,7 @@ public class DataNode extends Configured */ private synchronized void checkDatanodeUuid() throws IOException { if (storage.getDatanodeUuid() == null) { - storage.setDatanodeUuid(UUID.randomUUID().toString()); + storage.setDatanodeUuid(generateUuid()); storage.writeAll(); LOG.info("Generated and persisted new Datanode UUID " + storage.getDatanodeUuid()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java index 51fb9fec79c..94b625e2d71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java @@ -17,6 +17,14 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; @@ -25,28 +33,40 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.protocol.*; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; +import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataStorage; -import org.apache.hadoop.hdfs.server.protocol.*; +import org.apache.hadoop.hdfs.server.protocol.BlockCommand; +import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; +import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; +import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; +import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; +import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.net.DNS; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.Groups; -import org.apache.hadoop.util.*; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.util.VersionInfo; import org.apache.log4j.Level; import org.apache.log4j.LogManager; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.EnumSet; -import java.util.List; - /** * Main class for a series of name-node benchmarks. * @@ -817,13 +837,12 @@ public class NNThroughputBenchmark implements Tool { dnRegistration = new DatanodeRegistration( new DatanodeID(DNS.getDefaultIP("default"), DNS.getDefaultHost("default", "default"), - "", getNodePort(dnIdx), + DataNode.generateUuid(), getNodePort(dnIdx), DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT), new DataStorage(nsInfo), new ExportedBlockKeys(), VersionInfo.getVersion()); - // TODO: Fix NNThroughputBenchmark. // register datanode dnRegistration = nameNodeProto.registerDatanode(dnRegistration); //first block reports From 06d24efed0fef706a62b508cefc072578460f4c7 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 8 Nov 2013 01:01:18 +0000 Subject: [PATCH 37/76] HDFS-5475. NN incorrectly tracks more than one replica per DN. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1539890 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 4 ++++ .../hdfs/server/blockmanagement/BlockInfo.java | 16 +++++++++++++--- .../BlockInfoUnderConstruction.java | 17 +++++++++++++++-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index ab4303e7ece..87f89547584 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -84,3 +84,7 @@ IMPROVEMENTS: HDFS-5472. Fix TestDatanodeManager, TestSafeMode and TestNNThroughputBenchmark (Contributed by szetszwo) + + HDFS-5475. NN incorrectly tracks more than one replica per DN. (Arpit + Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java index c3fb37e2d5f..706042f580e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java @@ -195,14 +195,24 @@ public class BlockInfo extends Block implements LightWeightGSet.LinkedElement { * Add a {@link DatanodeStorageInfo} location for a block */ boolean addStorage(DatanodeStorageInfo storage) { - if(findStorageInfo(storage) >= 0) // the node is already there - return false; + boolean added = true; + int idx = findDatanode(storage.getDatanodeDescriptor()); + if(idx >= 0) { + if (getStorageInfo(idx) == storage) { // the storage is already there + return false; + } else { + // The block is on the DN but belongs to a different storage. + // Update our state. + removeStorage(storage); + added = false; // Just updating storage. Return false. + } + } // find the last null node int lastNode = ensureCapacity(1); setStorageInfo(lastNode, storage); setNext(lastNode, null); setPrevious(lastNode, null); - return true; + return added; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java index e580c542969..a2202feefba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java @@ -297,9 +297,22 @@ public class BlockInfoUnderConstruction extends BlockInfo { void addReplicaIfNotPresent(DatanodeStorageInfo storage, Block block, ReplicaState rState) { - for(ReplicaUnderConstruction r : replicas) - if(r.getExpectedStorageLocation() == storage) + Iterator it = replicas.iterator(); + while (it.hasNext()) { + ReplicaUnderConstruction r = it.next(); + if(r.getExpectedStorageLocation() == storage) { return; + } else if (r.getExpectedStorageLocation().getDatanodeDescriptor() == + storage.getDatanodeDescriptor()) { + + // The Datanode reported that the block is on a different storage + // than the one chosen by BlockPlacementPolicy. This can occur as + // we allow Datanodes to choose the target storage. Update our + // state by removing the stale entry and adding a new one. + it.remove(); + break; + } + } replicas.add(new ReplicaUnderConstruction(block, storage, rState)); } From 90d1b1b6a49c28d35c60c9ed6fd2f777faaebd81 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sat, 9 Nov 2013 00:19:48 +0000 Subject: [PATCH 38/76] HDFS-5481. Fix TestDataNodeVolumeFailure in branch HDFS-2832. (Contributed by Junping Du) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1540228 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 5 +++- .../datanode/TestDataNodeVolumeFailure.java | 27 ++++++++++++++----- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 87f89547584..50be9340354 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -85,6 +85,9 @@ IMPROVEMENTS: HDFS-5472. Fix TestDatanodeManager, TestSafeMode and TestNNThroughputBenchmark (Contributed by szetszwo) - HDFS-5475. NN incorrectly tracks more than one replica per DN. (Arpit + HDFS-5475. NN incorrectly tracks more than one replica per DN. (Arpit Agarwal) + HDFS-5481. Fix TestDataNodeVolumeFailure in branch HDFS-2832. (Contributed + by Junping Du) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java index 706916d8705..30e23420ac9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java @@ -42,11 +42,13 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.net.TcpPeerServer; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; @@ -151,13 +153,24 @@ public class TestDataNodeVolumeFailure { DataNode dn = cluster.getDataNodes().get(1); //corresponds to dir data3 String bpid = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(bpid); - final StorageBlockReport[] report = { - new StorageBlockReport( - new DatanodeStorage(dnR.getDatanodeUuid()), - DataNodeTestUtils.getFSDataset(dn).getBlockReport(bpid - ).getBlockListAsLongs()) - }; - cluster.getNameNodeRpc().blockReport(dnR, bpid, report); + + Map perVolumeBlockLists = + dn.getFSDataset().getBlockReports(bpid); + + // Send block report + StorageBlockReport[] reports = + new StorageBlockReport[perVolumeBlockLists.size()]; + + int reportIndex = 0; + for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { + String storageID = kvPair.getKey(); + BlockListAsLongs blockList = kvPair.getValue(); + DatanodeStorage dnStorage = new DatanodeStorage(storageID); + reports[reportIndex++] = + new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs()); + } + + cluster.getNameNodeRpc().blockReport(dnR, bpid, reports); // verify number of blocks and files... verify(filename, filesize); From 907fb15ee8c150e5ecc0560b7374441c57a84122 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sun, 10 Nov 2013 20:59:32 +0000 Subject: [PATCH 39/76] HDFS-5480. Update Balancer for HDFS-2832. (Contributed by szetszwo) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1540547 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 + ...atanodeProtocolServerSideTranslatorPB.java | 12 +--- .../hadoop/hdfs/protocolPB/PBHelper.java | 40 +++++++++---- .../hadoop/hdfs/server/balancer/Balancer.java | 58 ++++++++----------- .../server/blockmanagement/BlockManager.java | 24 +++++--- .../hdfs/server/datanode/DataXceiver.java | 3 +- .../server/protocol/BlocksWithLocations.java | 32 ++++++++-- .../hadoop-hdfs/src/main/proto/hdfs.proto | 3 +- .../apache/hadoop/hdfs/MiniDFSCluster.java | 11 +++- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 6 +- .../hdfs/server/balancer/TestBalancer.java | 22 ++++++- .../server/datanode/SimulatedFSDataset.java | 27 ++++++--- 12 files changed, 155 insertions(+), 85 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 50be9340354..b82839cd1c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -91,3 +91,5 @@ IMPROVEMENTS: HDFS-5481. Fix TestDataNodeVolumeFailure in branch HDFS-2832. (Contributed by Junping Du) + HDFS-5480. Update Balancer for HDFS-2832. (Contributed by szetszwo) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java index 1bf95678384..6f454e8b77b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReportBadBlo import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReportBadBlocksResponseProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageBlockReportProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageReceivedDeletedBlocksProto; -import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageReportProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionRequestProto; @@ -57,7 +56,6 @@ import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; -import com.google.common.primitives.Longs; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -103,14 +101,8 @@ public class DatanodeProtocolServerSideTranslatorPB implements HeartbeatRequestProto request) throws ServiceException { HeartbeatResponse response; try { - List list = request.getReportsList(); - StorageReport[] report = new StorageReport[list.size()]; - int i = 0; - for (StorageReportProto p : list) { - report[i++] = new StorageReport(p.getStorageUuid(), p.getFailed(), - p.getCapacity(), p.getDfsUsed(), p.getRemaining(), - p.getBlockPoolUsed()); - } + final StorageReport[] report = PBHelper.convertStorageReports( + request.getReportsList()); response = impl.sendHeartbeat(PBHelper.convert(request.getRegistration()), report, request.getDnCacheCapacity(), request.getDnCacheUsed(), request.getXmitsInProgress(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index ad660f5356f..c9673bea68b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -25,41 +25,40 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.List; -import com.google.common.base.Preconditions; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.proto.HAServiceProtocolProtos; import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.PathBasedCacheDirective; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.protocol.PathBasedCacheDirective; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; +import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateFlagProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeReportTypeProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsStatsResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SafeModeActionProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.PathBasedCacheDirectiveInfoProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SafeModeActionProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BalancerBandwidthCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockIdCommandProto; @@ -98,8 +97,6 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.File import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto.Builder; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlocksProto; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeCommandProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto.NamenodeRoleProto; @@ -113,6 +110,8 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryListingProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryStatusProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageInfoProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto; import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto; import org.apache.hadoop.hdfs.security.token.block.BlockKey; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; @@ -288,12 +287,16 @@ public class PBHelper { public static BlockWithLocationsProto convert(BlockWithLocations blk) { return BlockWithLocationsProto.newBuilder() .setBlock(convert(blk.getBlock())) + .addAllDatanodeUuids(Arrays.asList(blk.getDatanodeUuids())) .addAllStorageUuids(Arrays.asList(blk.getStorageIDs())).build(); } public static BlockWithLocations convert(BlockWithLocationsProto b) { - return new BlockWithLocations(convert(b.getBlock()), b.getStorageUuidsList() - .toArray(new String[0])); + final List datanodeUuids = b.getDatanodeUuidsList(); + final List storageUuids = b.getStorageUuidsList(); + return new BlockWithLocations(convert(b.getBlock()), + datanodeUuids.toArray(new String[datanodeUuids.size()]), + storageUuids.toArray(new String[storageUuids.size()])); } public static BlocksWithLocationsProto convert(BlocksWithLocations blks) { @@ -1535,6 +1538,21 @@ public class PBHelper { return builder.build(); } + public static StorageReport convert(StorageReportProto p) { + return new StorageReport(p.getStorageUuid(), p.getFailed(), + p.getCapacity(), p.getDfsUsed(), p.getRemaining(), + p.getBlockPoolUsed()); + } + + public static StorageReport[] convertStorageReports( + List list) { + final StorageReport[] report = new StorageReport[list.size()]; + for (int i = 0; i < report.length; i++) { + report[i] = convert(list.get(i)); + } + return report; + } + public static JournalInfo convert(JournalInfoProto info) { int lv = info.hasLayoutVersion() ? info.getLayoutVersion() : 0; int nsID = info.hasNamespaceID() ? info.getNamespaceID() : 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 23d64b4c4c7..983004dca11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.server.balancer; import static com.google.common.base.Preconditions.checkArgument; - import static org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed; import java.io.BufferedInputStream; @@ -221,9 +220,9 @@ public class Balancer { private Map globalBlockList = new HashMap(); private MovedBlocks movedBlocks = new MovedBlocks(); - // Map storage IDs to BalancerDatanodes - private Map datanodes - = new HashMap(); + /** Map (datanodeUuid -> BalancerDatanodes) */ + private final Map datanodeMap + = new HashMap(); private NetworkTopology cluster; @@ -241,6 +240,14 @@ public class Balancer { private PendingBlockMove() { } + @Override + public String toString() { + final Block b = block.getBlock(); + return b + " with size=" + b.getNumBytes() + " from " + + source.getDisplayName() + " to " + target.getDisplayName() + + " through " + proxySource.getDisplayName(); + } + /* choose a block & a proxy source for this pendingMove * whose source & target have already been chosen. * @@ -272,11 +279,7 @@ public class Balancer { if ( chooseProxySource() ) { movedBlocks.add(block); if (LOG.isDebugEnabled()) { - LOG.debug("Decided to move block "+ block.getBlockId() - +" with a length of "+StringUtils.byteDesc(block.getNumBytes()) - + " bytes from " + source.getDisplayName() - + " to " + target.getDisplayName() - + " using proxy source " + proxySource.getDisplayName() ); + LOG.debug("Decided to move " + this); } return true; } @@ -352,17 +355,9 @@ public class Balancer { sendRequest(out); receiveResponse(in); bytesMoved.inc(block.getNumBytes()); - LOG.info( "Moving block " + block.getBlock().getBlockId() + - " from "+ source.getDisplayName() + " to " + - target.getDisplayName() + " through " + - proxySource.getDisplayName() + - " is succeeded." ); + LOG.info("Successfully moved " + this); } catch (IOException e) { - LOG.warn("Error moving block "+block.getBlockId()+ - " from " + source.getDisplayName() + " to " + - target.getDisplayName() + " through " + - proxySource.getDisplayName() + - ": "+e.getMessage()); + LOG.warn("Failed to move " + this + ": " + e.getMessage()); } finally { IOUtils.closeStream(out); IOUtils.closeStream(in); @@ -414,9 +409,7 @@ public class Balancer { @Override public void run() { if (LOG.isDebugEnabled()) { - LOG.debug("Starting moving "+ block.getBlockId() + - " from " + proxySource.getDisplayName() + " to " + - target.getDisplayName()); + LOG.debug("Start moving " + PendingBlockMove.this); } dispatch(); } @@ -463,11 +456,6 @@ public class Balancer { return block; } - /* Return the block id */ - private long getBlockId() { - return block.getBlockId(); - } - /* Return the length of the block */ private long getNumBytes() { return block.getNumBytes(); @@ -674,10 +662,10 @@ public class Balancer { synchronized (block) { // update locations - for ( String storageID : blk.getStorageIDs() ) { - BalancerDatanode datanode = datanodes.get(storageID); + for (String datanodeUuid : blk.getDatanodeUuids()) { + final BalancerDatanode d = datanodeMap.get(datanodeUuid); if (datanode != null) { // not an unknown datanode - block.addLocation(datanode); + block.addLocation(d); } } } @@ -911,13 +899,13 @@ public class Balancer { datanodeS.utilization)*datanodeS.datanode.getCapacity()/100.0); } } - this.datanodes.put(datanode.getDatanodeUuid(), datanodeS); + datanodeMap.put(datanode.getDatanodeUuid(), datanodeS); } //logging logNodes(); - assert (this.datanodes.size() == + assert (this.datanodeMap.size() == overUtilizedDatanodes.size()+underUtilizedDatanodes.size()+ aboveAvgUtilizedDatanodes.size()+belowAvgUtilizedDatanodes.size()) : "Mismatched number of datanodes"; @@ -989,9 +977,9 @@ public class Balancer { // At last, match all remaining nodes chooseNodes(ANY_OTHER); - assert (datanodes.size() >= sources.size()+targets.size()) + assert (datanodeMap.size() >= sources.size()+targets.size()) : "Mismatched number of datanodes (" + - datanodes.size() + " total, " + + datanodeMap.size() + " total, " + sources.size() + " sources, " + targets.size() + " targets)"; @@ -1292,7 +1280,7 @@ public class Balancer { this.aboveAvgUtilizedDatanodes.clear(); this.belowAvgUtilizedDatanodes.clear(); this.underUtilizedDatanodes.clear(); - this.datanodes.clear(); + this.datanodeMap.clear(); this.sources.clear(); this.targets.clear(); this.policy.reset(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index b1ce880441d..dc540d4e675 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -694,17 +694,17 @@ public class BlockManager { /** * Get all valid locations of the block */ - private List getValidLocations(Block block) { - ArrayList machineSet = - new ArrayList(blocksMap.numNodes(block)); + private List getValidLocations(Block block) { + final List locations + = new ArrayList(blocksMap.numNodes(block)); for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) { final String storageID = storage.getStorageID(); // filter invalidate replicas if(!invalidateBlocks.contains(storageID, block)) { - machineSet.add(storageID); + locations.add(storage); } } - return machineSet; + return locations; } private List createLocatedBlockList(final BlockInfo[] blocks, @@ -2622,12 +2622,18 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block * return the length of the added block; 0 if the block is not added */ private long addBlock(Block block, List results) { - final List machineSet = getValidLocations(block); - if(machineSet.size() == 0) { + final List locations = getValidLocations(block); + if(locations.size() == 0) { return 0; } else { - results.add(new BlockWithLocations(block, - machineSet.toArray(new String[machineSet.size()]))); + final String[] datanodeUuids = new String[locations.size()]; + final String[] storageIDs = new String[datanodeUuids.length]; + for(int i = 0; i < locations.size(); i++) { + final DatanodeStorageInfo s = locations.get(i); + datanodeUuids[i] = s.getDatanodeDescriptor().getDatanodeUuid(); + storageIDs[i] = s.getStorageID(); + } + results.add(new BlockWithLocations(block, datanodeUuids, storageIDs)); return block.getNumBytes(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index b4da358bda6..24345841211 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -866,7 +866,8 @@ class DataXceiver extends Receiver implements Runnable { datanode.notifyNamenodeReceivedBlock( block, delHint, blockReceiver.getReplicaInfo().getStorageUuid()); - LOG.info("Moved " + block + " from " + peer.getRemoteAddressString()); + LOG.info("Moved " + block + " from " + peer.getRemoteAddressString() + + ", delHint=" + delHint); } catch (IOException ioe) { opStatus = ERROR; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java index 6c672b1c771..e4f34ad5fe3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.protocol; +import java.util.Arrays; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.protocol.Block; @@ -34,12 +36,14 @@ public class BlocksWithLocations { @InterfaceAudience.Private @InterfaceStability.Evolving public static class BlockWithLocations { - Block block; - String storageIDs[]; + final Block block; + final String[] datanodeUuids; + final String[] storageIDs; /** constructor */ - public BlockWithLocations(Block block, String[] storageIDs) { + public BlockWithLocations(Block block, String[] datanodeUuids, String[] storageIDs) { this.block = block; + this.datanodeUuids = datanodeUuids; this.storageIDs = storageIDs; } @@ -48,10 +52,30 @@ public class BlocksWithLocations { return block; } - /** get the block's locations */ + /** get the block's datanode locations */ + public String[] getDatanodeUuids() { + return datanodeUuids; + } + + /** get the block's storage locations */ public String[] getStorageIDs() { return storageIDs; } + + @Override + public String toString() { + final StringBuilder b = new StringBuilder(); + b.append(block); + if (datanodeUuids.length == 0) { + return b.append("[]").toString(); + } + + b.append(storageIDs[0]).append('@').append(datanodeUuids[0]); + for(int i = 1; i < datanodeUuids.length; i++) { + b.append(", ").append(storageIDs[i]).append("@").append(datanodeUuids[i]); + } + return b.append("]").toString(); + } } private BlockWithLocations[] blocks; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 3444c220020..b903008a2d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -355,7 +355,8 @@ message BlockProto { */ message BlockWithLocationsProto { required BlockProto block = 1; // Block - repeated string storageUuids = 2; // Datanodes with replicas of the block + repeated string datanodeUuids = 2; // Datanodes with replicas of the block + repeated string storageUuids = 3; // Storages with replicas of the block } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 29c0595a231..4f96b830a35 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -328,9 +328,10 @@ public class MiniDFSCluster { builder.nameNodePort, builder.nameNodeHttpPort); } - LOG.info("starting cluster with " + - builder.nnTopology.countNameNodes() + " namenodes."); - nameNodes = new NameNodeInfo[builder.nnTopology.countNameNodes()]; + final int numNameNodes = builder.nnTopology.countNameNodes(); + LOG.info("starting cluster: numNameNodes=" + numNameNodes + + ", numDataNodes=" + builder.numDataNodes); + nameNodes = new NameNodeInfo[numNameNodes]; initMiniDFSCluster(builder.conf, builder.numDataNodes, @@ -1920,12 +1921,14 @@ public class MiniDFSCluster { // Wait for expected number of datanodes to start if (dnInfo.length != numDataNodes) { + LOG.info("dnInfo.length != numDataNodes"); return true; } // if one of the data nodes is not fully started, continue to wait for (DataNodeProperties dn : dataNodes) { if (!dn.datanode.isDatanodeFullyStarted()) { + LOG.info("!dn.datanode.isDatanodeFullyStarted()"); return true; } } @@ -1934,6 +1937,7 @@ public class MiniDFSCluster { // using (capacity == 0) as proxy. for (DatanodeInfo dn : dnInfo) { if (dn.getCapacity() == 0) { + LOG.info("dn.getCapacity() == 0"); return true; } } @@ -1941,6 +1945,7 @@ public class MiniDFSCluster { // If datanode dataset is not initialized then wait for (DataNodeProperties dn : dataNodes) { if (DataNodeTestUtils.getFSDataset(dn.datanode) == null) { + LOG.info("DataNodeTestUtils.getFSDataset(dn.datanode) == null"); return true; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index 2b1817956f5..60c6d6bbbb3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -166,8 +166,10 @@ public class TestPBHelper { } private static BlockWithLocations getBlockWithLocations(int bid) { - return new BlockWithLocations(new Block(bid, 0, 1), new String[] { "dn1", - "dn2", "dn3" }); + final String[] datanodeUuids = {"dn1", "dn2", "dn3"}; + final String[] storageIDs = {"s1", "s2", "s3"}; + return new BlockWithLocations(new Block(bid, 0, 1), + datanodeUuids, storageIDs); } private void compare(BlockWithLocations locs1, BlockWithLocations locs2) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index 86dd3ab7589..f78eec13b25 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -31,6 +31,7 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -50,6 +51,7 @@ import org.apache.hadoop.hdfs.server.balancer.Balancer.Cli; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Tool; +import org.apache.log4j.Level; import org.junit.Test; /** @@ -58,7 +60,10 @@ import org.junit.Test; public class TestBalancer { private static final Log LOG = LogFactory.getLog( "org.apache.hadoop.hdfs.TestBalancer"); - + static { + ((Log4JLogger)Balancer.LOG).getLogger().setLevel(Level.ALL); + } + final static long CAPACITY = 500L; final static String RACK0 = "/rack0"; final static String RACK1 = "/rack1"; @@ -292,6 +297,16 @@ public class TestBalancer { } while (!balanced); } + String long2String(long[] array) { + if (array.length == 0) { + return ""; + } + StringBuilder b = new StringBuilder("[").append(array[0]); + for(int i = 1; i < array.length; i++) { + b.append(", ").append(array[i]); + } + return b.append("]").toString(); + } /** This test start a cluster with specified number of nodes, * and fills it to be 30% full (with a single file replicated identically * to all datanodes); @@ -308,6 +323,11 @@ public class TestBalancer { */ private void doTest(Configuration conf, long[] capacities, String[] racks, long newCapacity, String newRack, boolean useTool) throws Exception { + LOG.info("capacities = " + long2String(capacities)); + LOG.info("racks = " + Arrays.asList(racks)); + LOG.info("newCapacity= " + newCapacity); + LOG.info("newRack = " + newRack); + LOG.info("useTool = " + useTool); assertEquals(capacities.length, racks.length); int numOfDatanodes = capacities.length; cluster = new MiniDFSCluster.Builder(conf) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 07ae075a5ba..8870b23490c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -21,7 +21,11 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import javax.management.NotCompliantMBeanException; import javax.management.ObjectName; @@ -43,6 +47,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; import org.apache.hadoop.hdfs.server.datanode.fsdataset.RollingLogs; import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.io.IOUtils; @@ -319,15 +324,15 @@ public class SimulatedFSDataset implements FsDatasetSpi { private static class SimulatedStorage { private Map map = new HashMap(); - private final String storageUuid = "SimulatedStorage-UUID"; + private final String storageUuid = "SimulatedStroage-" + DatanodeStorage.newStorageID(); - private long capacity; // in bytes + private final long capacity; // in bytes synchronized long getFree() { return capacity - getUsed(); } - synchronized long getCapacity() { + long getCapacity() { return capacity; } @@ -383,9 +388,14 @@ public class SimulatedFSDataset implements FsDatasetSpi { return bpStorage; } - public String getStorageUuid() { + String getStorageUuid() { return storageUuid; } + + synchronized StorageReport getStorageReport(String bpid) { + return new StorageReport(getStorageUuid(), false, getCapacity(), + getUsed(), getFree(), map.get(bpid).getUsed()); + } } private final Map> blockMap @@ -400,8 +410,9 @@ public class SimulatedFSDataset implements FsDatasetSpi { } this.datanodeUuid = storage.getDatanodeUuid(); } else { - this.datanodeUuid = "unknownStorageId-" + UUID.randomUUID(); + this.datanodeUuid = "SimulatedDatanode-" + DataNode.generateUuid(); } + registerMBean(datanodeUuid); this.storage = new SimulatedStorage( conf.getLong(CONFIG_PROPERTY_CAPACITY, DEFAULT_CAPACITY)); @@ -478,7 +489,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { String bpid) { Map reports = new HashMap(); - reports.put("", getBlockReport(bpid)); + reports.put(storage.storageUuid, getBlockReport(bpid)); return reports; } @@ -1029,7 +1040,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { @Override public StorageReport[] getStorageReports(String bpid) { - return new StorageReport[0]; + return new StorageReport[] {storage.getStorageReport(bpid)}; } @Override From a161ff8f09dfc80bca213858d43bb1690cced09d Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 11 Nov 2013 06:40:06 +0000 Subject: [PATCH 40/76] HDFS-5486. Fix TestNameNodeMetrics for HDFS-2832. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1540619 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 ++ .../apache/hadoop/hdfs/MiniDFSCluster.java | 20 +++++++++++-------- .../namenode/metrics/TestNameNodeMetrics.java | 3 ++- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index b82839cd1c0..beaec8b1ce9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -93,3 +93,5 @@ IMPROVEMENTS: HDFS-5480. Update Balancer for HDFS-2832. (Contributed by szetszwo) + HDFS-5486. Fix TestNameNodeMetrics for HDFS-2832. (Arpit Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 4f96b830a35..98afa7aa79f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -124,6 +124,9 @@ public class MiniDFSCluster { public static final String DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY = DFS_NAMENODE_SAFEMODE_EXTENSION_KEY + ".testing"; + // Changing this value may break some tests that assume it is 2. + public static final int DIRS_PER_DATANODE = 2; + static { DefaultMetricsSystem.setMiniClusterMode(true); } /** @@ -1143,15 +1146,16 @@ public class MiniDFSCluster { // Set up datanode address setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig); if (manageDfsDirs) { - File dir1 = getInstanceStorageDir(i, 0); - File dir2 = getInstanceStorageDir(i, 1); - dir1.mkdirs(); - dir2.mkdirs(); - if (!dir1.isDirectory() || !dir2.isDirectory()) { - throw new IOException("Mkdirs failed to create directory for DataNode " - + i + ": " + dir1 + " or " + dir2); + StringBuilder sb = new StringBuilder(); + for (int j = 0; j < DIRS_PER_DATANODE; ++j) { + File dir = getInstanceStorageDir(i, j); + dir.mkdirs(); + if (!dir.isDirectory()) { + throw new IOException("Mkdirs failed to create directory for DataNode " + dir); + } + sb.append((j > 0 ? "," : "") + fileAsURI(dir)); } - String dirs = fileAsURI(dir1) + "," + fileAsURI(dir2); + String dirs = sb.toString(); dnConf.set(DFS_DATANODE_DATA_DIR_KEY, dirs); conf.set(DFS_DATANODE_DATA_DIR_KEY, dirs); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index b016230a8eb..b4f2a536dfb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -434,7 +434,8 @@ public class TestNameNodeMetrics { // We have one sync when the cluster starts up, just opening the journal assertCounter("SyncsNumOps", 1L, rb); // Each datanode reports in when the cluster comes up - assertCounter("BlockReportNumOps", (long)DATANODE_COUNT, rb); + assertCounter("BlockReportNumOps", + (long)DATANODE_COUNT*MiniDFSCluster.DIRS_PER_DATANODE, rb); // Sleep for an interval+slop to let the percentiles rollover Thread.sleep((PERCENTILES_INTERVAL+1)*1000); From 26f2bd67f2979b16af3593beb412103d254599c4 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 12 Nov 2013 00:48:55 +0000 Subject: [PATCH 41/76] HDFS-5491. Update editsStored for HDFS-2832. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1540903 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 + .../hadoop/hdfs/protocol/LayoutVersion.java | 4 +- .../hdfs/server/datanode/DataStorage.java | 4 +- .../ImageLoaderCurrent.java | 2 +- .../src/test/resources/editsStored | Bin 4647 -> 4612 bytes .../src/test/resources/editsStored.xml | 244 ++++++++---------- 6 files changed, 121 insertions(+), 135 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index beaec8b1ce9..dd71b294b9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -95,3 +95,5 @@ IMPROVEMENTS: HDFS-5486. Fix TestNameNodeMetrics for HDFS-2832. (Arpit Agarwal) + HDFS-5491. Update editsStored for HDFS-2832. (Arpit Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java index 3bdd3793757..094e4793cf2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java @@ -108,8 +108,8 @@ public class LayoutVersion { EDITLOG_SUPPORT_RETRYCACHE(-47, "Record ClientId and CallId in editlog to " + "enable rebuilding retry cache in case of HA failover"), CACHING(-48, "Support for cache pools and path-based caching"), - DATANODE_ID(-49, "UUID per Datanode and distinct StorageID per storage " - + "directory."); + ADD_DATANODE_AND_STORAGE_UUIDS(-49, "Replace StorageID with DatanodeUuid." + + " Use distinct StorageUuid per storage directory."); final int lv; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index cd84733288c..b391a78d4c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -291,8 +291,8 @@ public class DataStorage extends Storage { props.setProperty("layoutVersion", String.valueOf(layoutVersion)); props.setProperty("storageID", sd.getStorageUuid()); - if (LayoutVersion.supports(Feature.DATANODE_ID, layoutVersion) && - datanodeUuid != null) { + if (LayoutVersion.supports(Feature.ADD_DATANODE_AND_STORAGE_UUIDS, + layoutVersion) && datanodeUuid != null) { props.setProperty("datanodeUuid", datanodeUuid); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java index 25b128de38e..88687e70fed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java @@ -126,7 +126,7 @@ class ImageLoaderCurrent implements ImageLoader { new SimpleDateFormat("yyyy-MM-dd HH:mm"); private static int[] versions = { -16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36, -37, -38, -39, - -40, -41, -42, -43, -44, -45, -46, -47, -48 }; + -40, -41, -42, -43, -44, -45, -46, -47, -48, -49 }; private int imageVersion = 0; private final Map subtreeMap = new HashMap(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored index c1300fee9fae991d9d589c3192628cdc8c5378f8..65b85f77f2437e454c62bc6d66b570ab90ddd777 100644 GIT binary patch literal 4612 zcmcIodrTBZ82|Q`@i+3(IMGyZf3sU z?|aS6E~BHP$0LC>p)KOu)#3DqM%W^x*A}g9(RaO(o^v7j&dqpJfzZT?sv5;oW>;jVVsQy87ABiP01(nMr#DF+i4`=emR^?E z$@T`hCcxpUQ|#;PHdmd{0kN4YiZg1e6syxxP*|{P`O`&Nna^fqS%^@8@EVb5Re_aulQ*Bg-xy)1Rh}iVCSC(l?6acCHS_nWBfMW($Bt$ zTvMArA2pDKqt;fhloQ})E!WfuC))S*lYju|fTI`8)ib)@iu^*PM(Bx@L^_Ni%fI$-Q{$=-PfnRejV>D1qv;-*sV z31g?h;IPVdj!K)8pUe4Lm zk^J}-Kx@DG6pcKRxIOaP51$+v+sbM`n2@5Uul?D4di*xdPrLXE-Ssf(KB%9RB-HCD za|m~hF#7uP(jHp25XxNxdaKe<*kDyq4fjs=KVqbaVf@t~#HW4acma8$%I5~Uz7L`r z;hapXQ{tsj+Dz?hB56V)f=|HNnwJp@$}{f>ih4O_$9l>SlSxO?x{+czK|s(?oy3RIV55fr#~C ztBB%);`&d*pvG*XNb0Ub)0*D?nSV2Qx~s*3l+^t{GWcjNyc!G)zg0sB05DpcFp7vV zST=Cx zxr_65fThO|RmdQzitazo=b4By%*0YcF9dhbY&*x0XOT&9C@{6Lks;6aMpi?BRQ=TI zL)Zk!QwFzB#b=oCFluw~Ljv&csS5at@yx}8C_z`tHXPr^c;dZzSQ&WnxC_tU!`NM_ zzs(A79sx(s`=#_HPDkprEIJLN`{ALkdf7YA67JEClr>%! zkI5;udn5N^PcOdvTInQWCZrH4>A^_OQb%KLO_jC!KGbSBnkak|*M(6oX+~9OjxmIi zrh-Q^&sO@3T`sFeFG=V-4F$UuiOPApRsn@dNYBkW-#X>X-dmwmHSK{^j~(dSwIhtG zE*+n$>&?U4VyVh9L!Hf5QzzSL{)1sDkZY9!Te(sc_D(N5GV>@0_^=rtyLcnN`za@No>&m1K;{8Ir6+?r O=h9CP$Uok>JLq4f0Vd!8 literal 4647 zcmcIo4^UJ^82{cq;5ZOP5L5y^B{f8Z<4*{bK0FZtfl@QG8P4N)B5`+*?hcTd%#j&M zn>0%+btbdM)M=cCbcz`?@lWbhV>M=zR@%^?8AK2iWJ;%K_wDYTKGq9obeVbI-tKX9wA!u<3!69@o8?v5;biqB#Po$3{2 zbym3K`Ve<*jqIv)DYZ4i9*Cbef3ewKB|ANqlG2jm+@}`h&3ne2XGt=prJGXIv$C=* zCNP*%GiRotKNEycJJ8Vk{py0W#-bDNZ+LstI|w>>d&1w*29*OKkXrB`;q$|d@bvZ% z)}m>tC5Vq1N|wT*)X5bT*1-)lWVfT~0$B_IJ#hTibNxcM&e%~X(jbC~l|%?nuBdX! zHjmEcFlb@9%|5G0{LP0s^9X&CBWOuEk{4kbOq(CpS;f|C5dr{=)+7uf zVobJRpYFI3NKsEI2|!%8cyT-UoO%HDowVDy`_F%@6Hd#CF;w z+^rV2(|AdCE=9ojEVwhtUPj_i5|1W;R+3Cq)80wjRX-*aH2x~*T;J28q&5I9?0_cDCrct@|tU> zv%F2whR;$HfYi}N6w&(ouZ2e93{$a{&=ugHi`#x-$kRxt*d3bP(7=!%(;}-2fLsGq z;utT#?IfiMjKTofSK4z=)Sdo)UzEdTgjwT>?!MNEoRwD(T30T zrbl|~A$rT{sr$xAoWX>xgzle3U!J+hkW)#g*t@wpVi!YB(;}-2fL!&e-Vinc@@0R- zMe{}@Zbl^?KO_J{-E#+l@nqmel+5ezhLs*?JdbO6m;^LDUcDK7j0tCB%}j5Z1ROo@ z)|)q>3Zlp78%``P>U_LpceJ4@UOF;8O?xkGr>rUlY7(#Wgz<@OeJ{J^%;FyC$W^C_ zcSuG_)QXw=GMC0oKTq6W%+u=$@Y%WNa~n#AC7OFk2?4o zyV*W59LGj|$K`m+^rWFiQS3EVm#0CX!FsLGjrU6hcb-KX%|9m7Lmp<|0iw5AU3QP+ z6aopTa14RyPjNvOzTRCnY7&iP4z&UVB$r!VNYOr2u6Oavvp&pFUlGVtV%|C|U3|4z zzNY=N$+R_>^>TqqR>h?ZOrb;O{vU1Wt=P6?IvC8PA{k(O@+zyVBFQdW-EtE8h(q6w zu7)I!?DpOhYtUGndu(w*aiYQ$^1wf$3wOMA;ZLK=0Wc3+2_R%*pHX2aP9EtLZ}jiG z^f5be=KGlmmV=6dj?Wupl@F!$ZDp?c_viXk;a>wRlm#@DK37kO5>0+U!SLQ4epWtg z0s3!*rNruxOOy(E%e6@vjIfXr_MceOQpgC4;BE&$^U;i)u)28yJ8Kux@WG!uT1u>i h#sQ8de9KIb&_^CFVn0tSo7^}1>30egPdycs^B;W6bL{{C diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml index ed2f2b74aca..04808bde94f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml @@ -1,6 +1,6 @@ - -48 + -49 OP_START_LOG_SEGMENT @@ -13,8 +13,8 @@ 2 1 - 1381946377599 - 4f37c0db7342fb35 + 1384813583491 + b07b4248e233f9d8 @@ -24,8 +24,8 @@ 3 2 - 1381946377609 - 471d4ddd00402ba6 + 1384813583498 + 027ad6a5f9493c95 @@ -37,18 +37,18 @@ 16386 /file_create_u\0001;F431 1 - 1381255179312 - 1381255179312 + 1384122384704 + 1384122384704 512 - DFSClient_NONMAPREDUCE_-134124999_1 + DFSClient_NONMAPREDUCE_1053090993_1 127.0.0.1 - andrew + aagarwal supergroup 420 - 27ac79f0-d378-4933-824b-c2a188968d97 - 8 + 23253508-4e0a-4723-ae53-c964e27e7522 + 10 @@ -59,13 +59,13 @@ 0 /file_create_u\0001;F431 1 - 1381255179355 - 1381255179312 + 1384122384804 + 1384122384704 512 - andrew + aagarwal supergroup 420 @@ -78,9 +78,9 @@ 0 /file_create_u\0001;F431 /file_moved - 1381255179373 - 27ac79f0-d378-4933-824b-c2a188968d97 - 10 + 1384122384901 + 23253508-4e0a-4723-ae53-c964e27e7522 + 12 @@ -89,9 +89,9 @@ 7 0 /file_moved - 1381255179397 - 27ac79f0-d378-4933-824b-c2a188968d97 - 11 + 1384122384912 + 23253508-4e0a-4723-ae53-c964e27e7522 + 13 @@ -101,9 +101,9 @@ 0 16387 /directory_mkdir - 1381255179424 + 1384122385030 - andrew + aagarwal supergroup 493 @@ -136,8 +136,8 @@ 12 /directory_mkdir snapshot1 - 27ac79f0-d378-4933-824b-c2a188968d97 - 16 + 23253508-4e0a-4723-ae53-c964e27e7522 + 18 @@ -147,8 +147,8 @@ /directory_mkdir snapshot1 snapshot2 - 27ac79f0-d378-4933-824b-c2a188968d97 - 17 + 23253508-4e0a-4723-ae53-c964e27e7522 + 19 @@ -157,8 +157,8 @@ 14 /directory_mkdir snapshot2 - 27ac79f0-d378-4933-824b-c2a188968d97 - 18 + 23253508-4e0a-4723-ae53-c964e27e7522 + 20 @@ -169,18 +169,18 @@ 16388 /file_create_u\0001;F431 1 - 1381255179522 - 1381255179522 + 1384122385108 + 1384122385108 512 - DFSClient_NONMAPREDUCE_-134124999_1 + DFSClient_NONMAPREDUCE_1053090993_1 127.0.0.1 - andrew + aagarwal supergroup 420 - 27ac79f0-d378-4933-824b-c2a188968d97 - 19 + 23253508-4e0a-4723-ae53-c964e27e7522 + 21 @@ -191,13 +191,13 @@ 0 /file_create_u\0001;F431 1 - 1381255179531 - 1381255179522 + 1384122385110 + 1384122385108 512 - andrew + aagarwal supergroup 420 @@ -253,10 +253,10 @@ 0 /file_create_u\0001;F431 /file_moved - 1381255179602 + 1384122385129 NONE - 27ac79f0-d378-4933-824b-c2a188968d97 - 26 + 23253508-4e0a-4723-ae53-c964e27e7522 + 28 @@ -267,18 +267,18 @@ 16389 /file_concat_target 1 - 1381255179619 - 1381255179619 + 1384122385133 + 1384122385133 512 - DFSClient_NONMAPREDUCE_-134124999_1 + DFSClient_NONMAPREDUCE_1053090993_1 127.0.0.1 - andrew + aagarwal supergroup 420 - 27ac79f0-d378-4933-824b-c2a188968d97 - 28 + 23253508-4e0a-4723-ae53-c964e27e7522 + 30 @@ -388,8 +388,8 @@ 0 /file_concat_target 1 - 1381255179862 - 1381255179619 + 1384122385210 + 1384122385133 512 @@ -409,7 +409,7 @@ 1003 - andrew + aagarwal supergroup 420 @@ -423,18 +423,18 @@ 16390 /file_concat_0 1 - 1381255179876 - 1381255179876 + 1384122385213 + 1384122385213 512 - DFSClient_NONMAPREDUCE_-134124999_1 + DFSClient_NONMAPREDUCE_1053090993_1 127.0.0.1 - andrew + aagarwal supergroup 420 - 27ac79f0-d378-4933-824b-c2a188968d97 - 41 + 23253508-4e0a-4723-ae53-c964e27e7522 + 43 @@ -544,8 +544,8 @@ 0 /file_concat_0 1 - 1381255179957 - 1381255179876 + 1384122385243 + 1384122385213 512 @@ -565,7 +565,7 @@ 1006 - andrew + aagarwal supergroup 420 @@ -579,18 +579,18 @@ 16391 /file_concat_1 1 - 1381255179967 - 1381255179967 + 1384122385246 + 1384122385246 512 - DFSClient_NONMAPREDUCE_-134124999_1 + DFSClient_NONMAPREDUCE_1053090993_1 127.0.0.1 - andrew + aagarwal supergroup 420 - 27ac79f0-d378-4933-824b-c2a188968d97 - 53 + 23253508-4e0a-4723-ae53-c964e27e7522 + 55 @@ -700,8 +700,8 @@ 0 /file_concat_1 1 - 1381255180085 - 1381255179967 + 1384122385274 + 1384122385246 512 @@ -721,7 +721,7 @@ 1009 - andrew + aagarwal supergroup 420 @@ -733,13 +733,13 @@ 56 0 /file_concat_target - 1381255180099 + 1384122385277 /file_concat_0 /file_concat_1 - 27ac79f0-d378-4933-824b-c2a188968d97 - 64 + 23253508-4e0a-4723-ae53-c964e27e7522 + 66 @@ -750,15 +750,15 @@ 16392 /file_symlink /file_concat_target - 1381255180116 - 1381255180116 + 1384122385281 + 1384122385281 - andrew + aagarwal supergroup 511 - 27ac79f0-d378-4933-824b-c2a188968d97 - 65 + 23253508-4e0a-4723-ae53-c964e27e7522 + 67 @@ -768,14 +768,14 @@ HDFS_DELEGATION_TOKEN 1 - andrew + aagarwal JobTracker - 1381255180128 - 1381859980128 + 1384122385285 + 1384727185285 2 - 1381341580128 + 1384208785285 @@ -785,14 +785,14 @@ HDFS_DELEGATION_TOKEN 1 - andrew + aagarwal JobTracker - 1381255180128 - 1381859980128 + 1384122385285 + 1384727185285 2 - 1381341580177 + 1384208785358 @@ -802,11 +802,11 @@ HDFS_DELEGATION_TOKEN 1 - andrew + aagarwal JobTracker - 1381255180128 - 1381859980128 + 1384122385285 + 1384727185285 2 @@ -817,13 +817,13 @@ 61 poolparty - andrew - andrew + aagarwal + staff 493 100 - 27ac79f0-d378-4933-824b-c2a188968d97 - 75 + 23253508-4e0a-4723-ae53-c964e27e7522 + 71 @@ -835,8 +835,8 @@ party 448 1989 - 27ac79f0-d378-4933-824b-c2a188968d97 - 76 + 23253508-4e0a-4723-ae53-c964e27e7522 + 72 @@ -847,8 +847,8 @@ /bar 1 poolparty - 27ac79f0-d378-4933-824b-c2a188968d97 - 77 + 23253508-4e0a-4723-ae53-c964e27e7522 + 73 @@ -856,9 +856,9 @@ 64 1 - 2 - - -2 + /bar2 + 23253508-4e0a-4723-ae53-c964e27e7522 + 74 @@ -866,8 +866,8 @@ 65 1 - 27ac79f0-d378-4933-824b-c2a188968d97 - 78 + 23253508-4e0a-4723-ae53-c964e27e7522 + 75 @@ -875,8 +875,8 @@ 66 poolparty - 27ac79f0-d378-4933-824b-c2a188968d97 - 79 + 23253508-4e0a-4723-ae53-c964e27e7522 + 76 @@ -887,18 +887,18 @@ 16393 /hard-lease-recovery-test 1 - 1381255180288 - 1381255180288 + 1384122385394 + 1384122385394 512 - DFSClient_NONMAPREDUCE_-134124999_1 + DFSClient_NONMAPREDUCE_1053090993_1 127.0.0.1 - andrew + aagarwal supergroup 420 - 27ac79f0-d378-4933-824b-c2a188968d97 - 74 + 23253508-4e0a-4723-ae53-c964e27e7522 + 77 @@ -954,23 +954,7 @@ OP_REASSIGN_LEASE 73 - DFSClient_NONMAPREDUCE_-134124999_1 - /hard-lease-recovery-test - HDFS_NameNode - - - - OP_SET_GENSTAMP_V2 - - 74 - 1012 - - - - OP_REASSIGN_LEASE - - 75 - HDFS_NameNode + DFSClient_NONMAPREDUCE_1053090993_1 /hard-lease-recovery-test HDFS_NameNode @@ -978,23 +962,23 @@ OP_CLOSE - 76 + 74 0 0 /hard-lease-recovery-test 1 - 1381255185142 - 1381255180288 + 1384122387627 + 1384122385394 512 1073741834 11 - 1012 + 1011 - andrew + aagarwal supergroup 420 @@ -1003,7 +987,7 @@ OP_END_LOG_SEGMENT - 77 + 75 From e5b3171c3ba07e18d07df2765ded907e28c1b252 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 12 Nov 2013 02:36:24 +0000 Subject: [PATCH 42/76] HDFS-5494. Fix findbugs warnings for HDFS-2832. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1540924 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 + .../hdfs/server/datanode/BPServiceActor.java | 65 +++++++++++-------- .../hdfs/server/datanode/DataStorage.java | 4 +- 3 files changed, 42 insertions(+), 29 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index dd71b294b9e..227e8eb4c9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -97,3 +97,5 @@ IMPROVEMENTS: HDFS-5491. Update editsStored for HDFS-2832. (Arpit Agarwal) + HDFS-5494. Fix findbugs warnings for HDFS-2832. (Arpit Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 5df81153ef8..a10768c3d27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -98,7 +98,7 @@ class BPServiceActor implements Runnable { * reported to the NN. Access should be synchronized on this object. */ private final Map - pendingIncrementalBRperStorage = Maps.newConcurrentMap(); + pendingIncrementalBRperStorage = Maps.newHashMap(); private volatile int pendingReceivedRequests = 0; private volatile boolean shouldServiceRun = true; @@ -266,43 +266,54 @@ class BPServiceActor implements Runnable { } /** - * Report received blocks and delete hints to the Namenode + * Report received blocks and delete hints to the Namenode for each + * storage. + * * @throws IOException */ private void reportReceivedDeletedBlocks() throws IOException { - // For each storage, check if there are newly received blocks and if - // so then send an incremental report to the NameNode. - for (Map.Entry entry : - pendingIncrementalBRperStorage.entrySet()) { - final String storageUuid = entry.getKey(); - final PerStoragePendingIncrementalBR perStorageMap = entry.getValue(); - ReceivedDeletedBlockInfo[] receivedAndDeletedBlockArray = null; - // TODO: We can probably use finer-grained synchronization now. - synchronized (pendingIncrementalBRperStorage) { + + // Generate a list of the pending reports for each storage under the lock + Map blockArrays = Maps.newHashMap(); + synchronized (pendingIncrementalBRperStorage) { + for (Map.Entry entry : + pendingIncrementalBRperStorage.entrySet()) { + final String storageUuid = entry.getKey(); + final PerStoragePendingIncrementalBR perStorageMap = entry.getValue(); + ReceivedDeletedBlockInfo[] receivedAndDeletedBlockArray = null; + if (perStorageMap.getBlockInfoCount() > 0) { // Send newly-received and deleted blockids to namenode receivedAndDeletedBlockArray = perStorageMap.dequeueBlockInfos(); pendingReceivedRequests -= receivedAndDeletedBlockArray.length; + blockArrays.put(storageUuid, receivedAndDeletedBlockArray); } } + } - if (receivedAndDeletedBlockArray != null) { - StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( - storageUuid, receivedAndDeletedBlockArray) }; - boolean success = false; - try { - bpNamenode.blockReceivedAndDeleted(bpRegistration, bpos.getBlockPoolId(), - report); - success = true; - } finally { + // Send incremental block reports to the Namenode outside the lock + for (Map.Entry entry : + blockArrays.entrySet()) { + final String storageUuid = entry.getKey(); + final ReceivedDeletedBlockInfo[] rdbi = entry.getValue(); + + StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( + storageUuid, rdbi) }; + boolean success = false; + try { + bpNamenode.blockReceivedAndDeleted(bpRegistration, + bpos.getBlockPoolId(), report); + success = true; + } finally { + if (!success) { synchronized (pendingIncrementalBRperStorage) { - if (!success) { - // If we didn't succeed in sending the report, put all of the - // blocks back onto our queue, but only in the case where we - // didn't put something newer in the meantime. - perStorageMap.putMissingBlockInfos(receivedAndDeletedBlockArray); - pendingReceivedRequests += perStorageMap.getBlockInfoCount(); - } + // If we didn't succeed in sending the report, put all of the + // blocks back onto our queue, but only in the case where we + // didn't put something newer in the meantime. + PerStoragePendingIncrementalBR perStorageMap = + pendingIncrementalBRperStorage.get(storageUuid); + perStorageMap.putMissingBlockInfos(rdbi); + pendingReceivedRequests += perStorageMap.getBlockInfoCount(); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index b391a78d4c9..aad857c0eaa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -293,7 +293,7 @@ public class DataStorage extends Storage { if (LayoutVersion.supports(Feature.ADD_DATANODE_AND_STORAGE_UUIDS, layoutVersion) && datanodeUuid != null) { - props.setProperty("datanodeUuid", datanodeUuid); + props.setProperty("datanodeUuid", getDatanodeUuid()); } // Set NamespaceID in version before federation @@ -348,7 +348,7 @@ public class DataStorage extends Storage { } else if (getDatanodeUuid().compareTo(dnUuid) != 0) { throw new InconsistentFSStateException(sd.getRoot(), "Root " + sd.getRoot() + ": DatanodeUuid=" + dnUuid + - ", does not match " + datanodeUuid + " from other" + + ", does not match " + getDatanodeUuid() + " from other" + " StorageDirectory."); } } From 46cbce9af1272ce0eb6e300f96a1a8d4b08e23e3 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 13 Nov 2013 02:00:41 +0000 Subject: [PATCH 43/76] HDFS-5508. Fix compilation error after merge. (Contributed by szetszwo) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1541352 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 ++ .../src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 227e8eb4c9e..1e33fa0e0a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -99,3 +99,5 @@ IMPROVEMENTS: HDFS-5494. Fix findbugs warnings for HDFS-2832. (Arpit Agarwal) + HDFS-5508. Fix compilation error after merge. (Contributed by szetszwo) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 58fafdee07f..63fbda61d6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -414,7 +414,7 @@ public class JsonUtil { (Object[])m.get("cachedLocations")); final LocatedBlock locatedblock = new LocatedBlock(b, locations, - startOffset, isCorrupt, cachedLocations); + null, null, startOffset, isCorrupt, cachedLocations); locatedblock.setBlockToken(toBlockToken((Map)m.get("blockToken"))); return locatedblock; } From d06a782c1e4243e36b486f66e66cdcc91ec6e275 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 13 Nov 2013 02:59:46 +0000 Subject: [PATCH 44/76] HDFS-5501. Fix pendingReceivedRequests tracking in BPServiceActor. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1541371 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 +++ .../hdfs/server/datanode/BPServiceActor.java | 18 +++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 1e33fa0e0a8..485f782c7fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -101,3 +101,6 @@ IMPROVEMENTS: HDFS-5508. Fix compilation error after merge. (Contributed by szetszwo) + HDFS-5501. Fix pendingReceivedRequests tracking in BPServiceActor. (Arpit + Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index a10768c3d27..81cf5f79fcf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -280,13 +280,14 @@ class BPServiceActor implements Runnable { pendingIncrementalBRperStorage.entrySet()) { final String storageUuid = entry.getKey(); final PerStoragePendingIncrementalBR perStorageMap = entry.getValue(); - ReceivedDeletedBlockInfo[] receivedAndDeletedBlockArray = null; if (perStorageMap.getBlockInfoCount() > 0) { // Send newly-received and deleted blockids to namenode - receivedAndDeletedBlockArray = perStorageMap.dequeueBlockInfos(); - pendingReceivedRequests -= receivedAndDeletedBlockArray.length; - blockArrays.put(storageUuid, receivedAndDeletedBlockArray); + ReceivedDeletedBlockInfo[] rdbi = perStorageMap.dequeueBlockInfos(); + pendingReceivedRequests = + (pendingReceivedRequests > rdbi.length ? + (pendingReceivedRequests - rdbi.length) : 0); + blockArrays.put(storageUuid, rdbi); } } } @@ -312,8 +313,7 @@ class BPServiceActor implements Runnable { // didn't put something newer in the meantime. PerStoragePendingIncrementalBR perStorageMap = pendingIncrementalBRperStorage.get(storageUuid); - perStorageMap.putMissingBlockInfos(rdbi); - pendingReceivedRequests += perStorageMap.getBlockInfoCount(); + pendingReceivedRequests += perStorageMap.putMissingBlockInfos(rdbi); } } } @@ -859,13 +859,17 @@ class BPServiceActor implements Runnable { * Add blocks from blockArray to pendingIncrementalBR, unless the * block already exists in pendingIncrementalBR. * @param blockArray list of blocks to add. + * @return the number of missing blocks that we added. */ - void putMissingBlockInfos(ReceivedDeletedBlockInfo[] blockArray) { + int putMissingBlockInfos(ReceivedDeletedBlockInfo[] blockArray) { + int blocksPut = 0; for (ReceivedDeletedBlockInfo rdbi : blockArray) { if (!pendingIncrementalBR.containsKey(rdbi.getBlock().getBlockId())) { pendingIncrementalBR.put(rdbi.getBlock().getBlockId(), rdbi); + ++blocksPut; } } + return blocksPut; } /** From ea1c5ea0866fe6cccbae851e3eb8b9fd564258bf Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 14 Nov 2013 18:16:02 +0000 Subject: [PATCH 45/76] HDFS-5510. Fix a findbug warning in DataStorage.java on HDFS-2832 branch. (Contributed by Junping Du) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1542006 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 +++ .../org/apache/hadoop/hdfs/server/datanode/DataStorage.java | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 485f782c7fd..0bb1ae94019 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -104,3 +104,6 @@ IMPROVEMENTS: HDFS-5501. Fix pendingReceivedRequests tracking in BPServiceActor. (Arpit Agarwal) + HDFS-5510. Fix a findbug warning in DataStorage.java on HDFS-2832 branch. + (Junping Du via Arpit Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index aad857c0eaa..819c7342b48 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -291,9 +291,10 @@ public class DataStorage extends Storage { props.setProperty("layoutVersion", String.valueOf(layoutVersion)); props.setProperty("storageID", sd.getStorageUuid()); + String datanodeUuid = getDatanodeUuid(); if (LayoutVersion.supports(Feature.ADD_DATANODE_AND_STORAGE_UUIDS, layoutVersion) && datanodeUuid != null) { - props.setProperty("datanodeUuid", getDatanodeUuid()); + props.setProperty("datanodeUuid", datanodeUuid); } // Set NamespaceID in version before federation From cd768489f373ca101a0bfd285ec9b8695087fc42 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 15 Nov 2013 05:21:58 +0000 Subject: [PATCH 46/76] HDFS-5515. Fix TestDFSStartupVersions for HDFS-2832. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1542176 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 ++ .../hadoop/hdfs/server/datanode/DataStorage.java | 7 +++---- .../org/apache/hadoop/hdfs/TestDFSStartupVersions.java | 2 +- .../java/org/apache/hadoop/hdfs/UpgradeUtilities.java | 1 + .../server/namenode/TestListCorruptFileBlocks.java | 10 +++++----- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 0bb1ae94019..41d44079472 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -106,4 +106,6 @@ IMPROVEMENTS: HDFS-5510. Fix a findbug warning in DataStorage.java on HDFS-2832 branch. (Junping Du via Arpit Agarwal) + + HDFS-5515. Fix TestDFSStartupVersions for HDFS-2832. (Arpit Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index 819c7342b48..e03151c64b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -97,8 +97,8 @@ public class DataStorage extends Storage { public synchronized String getDatanodeUuid() { return datanodeUuid; } - - synchronized void setDatanodeUuid(String newDatanodeUuid) { + + public synchronized void setDatanodeUuid(String newDatanodeUuid) { this.datanodeUuid = newDatanodeUuid; } @@ -292,8 +292,7 @@ public class DataStorage extends Storage { props.setProperty("storageID", sd.getStorageUuid()); String datanodeUuid = getDatanodeUuid(); - if (LayoutVersion.supports(Feature.ADD_DATANODE_AND_STORAGE_UUIDS, - layoutVersion) && datanodeUuid != null) { + if (datanodeUuid != null) { props.setProperty("datanodeUuid", datanodeUuid); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStartupVersions.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStartupVersions.java index 797d5ca38c9..980876cb86d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStartupVersions.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStartupVersions.java @@ -237,7 +237,7 @@ public class TestDFSStartupVersions { * this iterations version 3-tuple * */ - @Test + @Test (timeout=300000) public void testVersions() throws Exception { UpgradeUtilities.initialize(); Configuration conf = UpgradeUtilities.initializeStorageStateConf(1, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java index 52854588877..362aac46003 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java @@ -454,6 +454,7 @@ public class UpgradeUtilities { public static void createDataNodeVersionFile(File[] parent, StorageInfo version, String bpid, String bpidToWrite) throws IOException { DataStorage storage = new DataStorage(version); + storage.setDatanodeUuid("FixedDatanodeUuid"); File[] versionFiles = new File[parent.length]; for (int i = 0; i < parent.length; i++) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java index 7c8a712a79b..2b1578ca1d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java @@ -55,7 +55,7 @@ public class TestListCorruptFileBlocks { static Log LOG = NameNode.stateChangeLog; /** check if nn.getCorruptFiles() returns a file that has corrupted blocks */ - @Test + @Test (timeout=300000) public void testListCorruptFilesCorruptedBlock() throws Exception { MiniDFSCluster cluster = null; Random random = new Random(); @@ -131,7 +131,7 @@ public class TestListCorruptFileBlocks { /** * Check that listCorruptFileBlocks works while the namenode is still in safemode. */ - @Test + @Test (timeout=300000) public void testListCorruptFileBlocksInSafeMode() throws Exception { MiniDFSCluster cluster = null; Random random = new Random(); @@ -262,7 +262,7 @@ public class TestListCorruptFileBlocks { } // deliberately remove blocks from a file and validate the list-corrupt-file-blocks API - @Test + @Test (timeout=300000) public void testlistCorruptFileBlocks() throws Exception { Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000); @@ -372,7 +372,7 @@ public class TestListCorruptFileBlocks { /** * test listCorruptFileBlocks in DistributedFileSystem */ - @Test + @Test (timeout=300000) public void testlistCorruptFileBlocksDFS() throws Exception { Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000); @@ -445,7 +445,7 @@ public class TestListCorruptFileBlocks { * Also, test that DFS.listCorruptFileBlocks can make multiple successive * calls. */ - @Test + @Test (timeout=300000) public void testMaxCorruptFiles() throws Exception { MiniDFSCluster cluster = null; try { From 2d9692a36dc3922d3411bff1af02e2f275addd92 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 20 Nov 2013 17:13:35 +0000 Subject: [PATCH 47/76] HDFS-5527. Fix TestUnderReplicatedBlocks on branch HDFS-2832. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1543885 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 + .../hdfs/server/datanode/BPServiceActor.java | 37 ++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 41d44079472..908f661dce4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -109,3 +109,5 @@ IMPROVEMENTS: HDFS-5515. Fix TestDFSStartupVersions for HDFS-2832. (Arpit Agarwal) + HDFS-5527. Fix TestUnderReplicatedBlocks on branch HDFS-2832. (Arpit Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 81cf5f79fcf..f72ed7a2eb2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -341,6 +341,27 @@ class BPServiceActor implements Runnable { return mapForStorage; } + /** + * Add a blockInfo for notification to NameNode. If another entry + * exists for the same block it is removed. + * + * Caller must synchronize access using pendingIncrementalBRperStorage. + * @param bInfo + * @param storageUuid + */ + void addPendingReplicationBlockInfo(ReceivedDeletedBlockInfo bInfo, + String storageUuid) { + // Make sure another entry for the same block is first removed. + // There may only be one such entry. + for (Map.Entry entry : + pendingIncrementalBRperStorage.entrySet()) { + if (entry.getValue().removeBlockInfo(bInfo)) { + break; + } + } + getIncrementalBRMapForStorage(storageUuid).putBlockInfo(bInfo); + } + /* * Informing the name node could take a long long time! Should we wait * till namenode is informed before responding with success to the @@ -349,7 +370,7 @@ class BPServiceActor implements Runnable { void notifyNamenodeBlockImmediately( ReceivedDeletedBlockInfo bInfo, String storageUuid) { synchronized (pendingIncrementalBRperStorage) { - getIncrementalBRMapForStorage(storageUuid).putBlockInfo(bInfo); + addPendingReplicationBlockInfo(bInfo, storageUuid); pendingReceivedRequests++; pendingIncrementalBRperStorage.notifyAll(); } @@ -358,7 +379,7 @@ class BPServiceActor implements Runnable { void notifyNamenodeDeletedBlock( ReceivedDeletedBlockInfo bInfo, String storageUuid) { synchronized (pendingIncrementalBRperStorage) { - getIncrementalBRMapForStorage(storageUuid).putBlockInfo(bInfo); + addPendingReplicationBlockInfo(bInfo, storageUuid); } } @@ -880,5 +901,17 @@ class BPServiceActor implements Runnable { void putBlockInfo(ReceivedDeletedBlockInfo blockInfo) { pendingIncrementalBR.put(blockInfo.getBlock().getBlockId(), blockInfo); } + + /** + * Remove pending incremental block report for a single block if it + * exists. + * + * @param blockInfo + * @return true if a report was removed, false if no report existed for + * the given block. + */ + boolean removeBlockInfo(ReceivedDeletedBlockInfo blockInfo) { + return (pendingIncrementalBR.remove(blockInfo.getBlock().getBlockId()) != null); + } } } From b33ff7b5f267d8d5657d303e61ebeca8292ecf6f Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 21 Nov 2013 22:07:46 +0000 Subject: [PATCH 48/76] HDFS-5547. Fix build break after merge from trunk to HDFS-2832. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1544342 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt | 6 +++++- .../java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java | 3 --- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 908f661dce4..5e9b5537d9c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -109,5 +109,9 @@ IMPROVEMENTS: HDFS-5515. Fix TestDFSStartupVersions for HDFS-2832. (Arpit Agarwal) - HDFS-5527. Fix TestUnderReplicatedBlocks on branch HDFS-2832. (Arpit Agarwal) + HDFS-5527. Fix TestUnderReplicatedBlocks on branch HDFS-2832. (Arpit + Agarwal) + + HDFS-5547. Fix build break after merge from trunk to HDFS-2832. (Arpit + Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index c654386923c..c75941365c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -55,8 +55,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.protocol.PathBasedCacheDirective; -import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; @@ -67,7 +65,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheP import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateFlagProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeReportTypeProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsStatsResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.PathBasedCacheDirectiveInfoProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SafeModeActionProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BalancerBandwidthCommandProto; From 97acde2d33967f7f870f7dfe96c6b558e6fe324b Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 22 Nov 2013 20:07:10 +0000 Subject: [PATCH 49/76] HDFS-5542. Fix TODO and clean up the code in HDFS-2832. (Contributed by szetszwo) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1544664 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 + .../org/apache/hadoop/hdfs/StorageType.java | 18 +-- .../hadoop/hdfs/protocol/DatanodeID.java | 13 +- .../hadoop/hdfs/protocol/DatanodeInfo.java | 4 +- .../hadoop/hdfs/protocol/LocatedBlock.java | 4 - .../hadoop/hdfs/protocolPB/PBHelper.java | 59 ++++---- .../server/blockmanagement/BlocksMap.java | 4 +- .../blockmanagement/DatanodeDescriptor.java | 16 +-- .../blockmanagement/DatanodeStorageInfo.java | 36 ++--- .../PendingReplicationBlocks.java | 4 +- .../hadoop/hdfs/server/common/Storage.java | 3 +- .../hdfs/server/datanode/BPOfferService.java | 3 +- .../hdfs/server/datanode/BPServiceActor.java | 5 +- .../hdfs/server/datanode/BlockReceiver.java | 4 +- .../hadoop/hdfs/server/datanode/DataNode.java | 135 ++++++++++++------ .../hdfs/server/datanode/DataStorage.java | 11 +- .../hdfs/server/datanode/DataXceiver.java | 13 +- .../server/datanode/DirectoryScanner.java | 1 - .../server/datanode/FinalizedReplica.java | 4 - .../hdfs/server/datanode/StorageLocation.java | 46 +++--- .../datanode/fsdataset/FsDatasetSpi.java | 12 +- .../fsdataset/impl/FsDatasetImpl.java | 18 +-- .../datanode/fsdataset/impl/FsVolumeList.java | 2 +- .../datanode/fsdataset/impl/ReplicaMap.java | 4 +- .../hdfs/server/protocol/DatanodeStorage.java | 11 +- .../src/main/proto/DatanodeProtocol.proto | 2 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 +- .../apache/hadoop/hdfs/MiniDFSCluster.java | 12 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 13 +- .../TestInjectionForSimulatedStorage.java | 16 ++- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 5 +- .../blockmanagement/BlockManagerTestUtil.java | 14 +- .../server/blockmanagement/TestBlockInfo.java | 14 +- .../blockmanagement/TestBlockManager.java | 8 +- .../TestHeartbeatHandling.java | 8 +- .../TestOverReplicatedBlocks.java | 2 +- .../TestReplicationPolicy.java | 2 +- .../TestReplicationPolicyWithNodeGroup.java | 2 +- .../hdfs/server/common/TestJspHelper.java | 42 +++--- .../server/datanode/SimulatedFSDataset.java | 9 +- .../server/datanode/TestBlockRecovery.java | 21 ++- .../hdfs/server/datanode/TestDataDirs.java | 31 ++-- 42 files changed, 324 insertions(+), 314 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index 5e9b5537d9c..beced410222 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -115,3 +115,6 @@ IMPROVEMENTS: HDFS-5547. Fix build break after merge from trunk to HDFS-2832. (Arpit Agarwal) + HDFS-5542. Fix TODO and clean up the code in HDFS-2832. (Contributed by + szetszwo) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java index 3cdd01b959a..3f250f86fa5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StorageType.java @@ -24,22 +24,12 @@ import org.apache.hadoop.classification.InterfaceStability; /** * Defines the types of supported storage media. The default storage * medium is assumed to be DISK. - * */ @InterfaceAudience.Public -@InterfaceStability.Evolving +@InterfaceStability.Unstable public enum StorageType { - DISK(1), - SSD(2); + DISK, + SSD; + public static StorageType DEFAULT = DISK; - - private final int storageType; - - StorageType(int medium) { - storageType = medium; - } - - public int getStorageType() { - return this.storageType; - } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java index deaf694e182..c781e5b9ece 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java @@ -21,6 +21,8 @@ package org.apache.hadoop.hdfs.protocol; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import com.google.common.annotations.VisibleForTesting; + /** * This class represents the primary identifier for a Datanode. * Datanodes are identified by how they can be contacted (hostname @@ -45,9 +47,11 @@ public class DatanodeID implements Comparable { private int infoSecurePort; // info server port private int ipcPort; // IPC server port - // UUID identifying a given datanode. For upgraded Datanodes this is the - // same as the StorageID that was previously used by this Datanode. For - // newly formatted Datanodes it is a UUID. + /** + * UUID identifying a given datanode. For upgraded Datanodes this is the + * same as the StorageID that was previously used by this Datanode. + * For newly formatted Datanodes it is a UUID. + */ private String datanodeUuid = null; public DatanodeID(DatanodeID from) { @@ -99,7 +103,8 @@ public class DatanodeID implements Comparable { return datanodeUuid; } - public void setDatanodeUuid(String datanodeUuid) { + @VisibleForTesting + public void setDatanodeUuidForTesting(String datanodeUuid) { this.datanodeUuid = datanodeUuid; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java index 4aa3618234f..0f12e8c0bf5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java @@ -124,13 +124,13 @@ public class DatanodeInfo extends DatanodeID implements Node { /** Constructor */ public DatanodeInfo(final String ipAddr, final String hostName, - final String DatanodeUuid, final int xferPort, final int infoPort, + final String datanodeUuid, final int xferPort, final int infoPort, final int infoSecurePort, final int ipcPort, final long capacity, final long dfsUsed, final long remaining, final long blockPoolUsed, final long cacheCapacity, final long cacheUsed, final long lastUpdate, final int xceiverCount, final String networkLocation, final AdminStates adminState) { - super(ipAddr, hostName, DatanodeUuid, xferPort, infoPort, + super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort, ipcPort); this.capacity = capacity; this.dfsUsed = dfsUsed; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java index 282151651fb..0e6dd125464 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java @@ -123,10 +123,6 @@ public class LocatedBlock { return locs; } - public void setStorageTypes(StorageType[] storageTypes) { - this.storageTypes = storageTypes; - } - public StorageType[] getStorageTypes() { return storageTypes; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index c75941365c1..040cffd35c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -614,7 +614,7 @@ public class PBHelper { } final String[] storageIDs = b.getStorageIDs(); if (storageIDs != null) { - builder.addAllStorageIDs(Arrays.asList(b.getStorageIDs())); + builder.addAllStorageIDs(Arrays.asList(storageIDs)); } return builder.setB(PBHelper.convert(b.getBlock())) @@ -630,22 +630,23 @@ public class PBHelper { targets[i] = PBHelper.convert(locs.get(i)); } - List storageTypesList = proto.getStorageTypesList(); - StorageType[] storageTypes = new StorageType[locs.size()]; - - - // The media should correspond to targets 1:1. If not then - // ignore the media information (left as default). - if ((storageTypesList != null) && - (storageTypesList.size() == locs.size())) { - for (int i = 0; i < storageTypesList.size(); ++i) { - storageTypes[i] = PBHelper.convertType(storageTypesList.get(i)); - } + final int storageTypesCount = proto.getStorageTypesCount(); + final StorageType[] storageTypes; + if (storageTypesCount == 0) { + storageTypes = null; + } else { + Preconditions.checkState(storageTypesCount == locs.size()); + storageTypes = convertStorageTypeProtos(proto.getStorageTypesList()); } final int storageIDsCount = proto.getStorageIDsCount(); - final String[] storageIDs = storageIDsCount == 0? null - : proto.getStorageIDsList().toArray(new String[storageIDsCount]); + final String[] storageIDs; + if (storageIDsCount == 0) { + storageIDs = null; + } else { + Preconditions.checkState(storageIDsCount == locs.size()); + storageIDs = proto.getStorageIDsList().toArray(new String[storageIDsCount]); + } // Set values from the isCached list, re-using references from loc List cachedLocs = new ArrayList(locs.size()); @@ -1498,23 +1499,15 @@ public class PBHelper { case SSD: return StorageTypeProto.SSD; default: - Preconditions.checkState( - false, - "Failed to update StorageTypeProto with new StorageType " + - type.toString()); - return StorageTypeProto.DISK; + throw new IllegalStateException( + "BUG: StorageType not found, type=" + type); } } public static DatanodeStorage convert(DatanodeStorageProto s) { - if (s.hasStorageType()) { - return new DatanodeStorage(s.getStorageUuid(), - PBHelper.convertState(s.getState()), - PBHelper.convertType(s.getStorageType())); - } else { - return new DatanodeStorage(s.getStorageUuid(), - PBHelper.convertState(s.getState())); - } + return new DatanodeStorage(s.getStorageUuid(), + PBHelper.convertState(s.getState()), + PBHelper.convertType(s.getStorageType())); } private static State convertState(StorageState state) { @@ -1534,10 +1527,20 @@ public class PBHelper { case SSD: return StorageType.SSD; default: - return StorageType.DEFAULT; + throw new IllegalStateException( + "BUG: StorageTypeProto not found, type=" + type); } } + private static StorageType[] convertStorageTypeProtos( + List storageTypesList) { + final StorageType[] storageTypes = new StorageType[storageTypesList.size()]; + for (int i = 0; i < storageTypes.length; ++i) { + storageTypes[i] = PBHelper.convertType(storageTypesList.get(i)); + } + return storageTypes; + } + public static StorageReportProto convert(StorageReport r) { StorageReportProto.Builder builder = StorageReportProto.newBuilder() .setBlockPoolUsed(r.getBlockPoolUsed()).setCapacity(r.getCapacity()) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java index ef6442d8d88..643dc8c2aaa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java @@ -115,7 +115,7 @@ class BlocksMap { /** * Searches for the block in the BlocksMap and - * returns {@link Iterable} that iterates through the nodes the block belongs to. + * returns {@link Iterable} of the storages the block belongs to. */ Iterable getStorages(Block b) { return getStorages(blocks.get(b)); @@ -123,7 +123,7 @@ class BlocksMap { /** * For a block that has already been retrieved from the BlocksMap - * returns {@link Iterable} that iterates through the nodes the block belongs to. + * returns {@link Iterable} of the storages the block belongs to. */ Iterable getStorages(final BlockInfo storedBlock) { return new Iterable() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 5d64e0ca235..5621c17400f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -41,8 +41,6 @@ import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.util.IntrusiveCollection; import org.apache.hadoop.util.Time; -import com.google.common.annotations.VisibleForTesting; - /** * This class extends the DatanodeInfo class with ephemeral information (eg * health, capacity, what blocks are associated with the Datanode) that is @@ -176,15 +174,6 @@ public class DatanodeDescriptor extends DatanodeInfo { */ private long lastCachingDirectiveSentTimeMs; - /** - * Head of the list of blocks on the datanode - */ - private volatile BlockInfo blockList = null; - /** - * Number of blocks on the datanode - */ - private int numBlocks = 0; - // isAlive == heartbeats.contains(this) // This is an optimization, because contains takes O(n) time on Arraylist public boolean isAlive = false; @@ -661,8 +650,7 @@ public class DatanodeDescriptor extends DatanodeInfo { return sb.toString(); } - @VisibleForTesting - public DatanodeStorageInfo updateStorage(DatanodeStorage s) { + DatanodeStorageInfo updateStorage(DatanodeStorage s) { synchronized (storageMap) { DatanodeStorageInfo storage = storageMap.get(s.getStorageID()); if (storage == null) { @@ -670,8 +658,6 @@ public class DatanodeDescriptor extends DatanodeInfo { " for DN " + getXferAddr()); storage = new DatanodeStorageInfo(this, s); storageMap.put(s.getStorageID(), storage); - } else { - storage.setState(s.getState()); } return storage; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index 363e4de4d24..914d58a93ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -99,7 +99,7 @@ public class DatanodeStorageInfo { private final DatanodeDescriptor dn; private final String storageID; private final StorageType storageType; - private State state; + private final State state; private long capacity; private long dfsUsed; @@ -128,7 +128,7 @@ public class DatanodeStorageInfo { */ private boolean blockContentsStale = true; - public DatanodeStorageInfo(DatanodeDescriptor dn, DatanodeStorage s) { + DatanodeStorageInfo(DatanodeDescriptor dn, DatanodeStorage s) { this.dn = dn; this.storageID = s.getStorageID(); this.storageType = s.getStorageType(); @@ -165,7 +165,7 @@ public class DatanodeStorageInfo { } @VisibleForTesting - public void setUtilization(long capacity, long dfsUsed, + public void setUtilizationForTesting(long capacity, long dfsUsed, long remaining, long blockPoolUsed) { this.capacity = capacity; this.dfsUsed = dfsUsed; @@ -173,41 +173,35 @@ public class DatanodeStorageInfo { this.blockPoolUsed = blockPoolUsed; } - public void setState(State s) { - this.state = s; - - // TODO: if goes to failed state cleanup the block list - } - - public State getState() { + State getState() { return this.state; } - public String getStorageID() { + String getStorageID() { return storageID; } - public StorageType getStorageType() { + StorageType getStorageType() { return storageType; } - public long getCapacity() { + long getCapacity() { return capacity; } - public long getDfsUsed() { + long getDfsUsed() { return dfsUsed; } - public long getRemaining() { + long getRemaining() { return remaining; } - public long getBlockPoolUsed() { + long getBlockPoolUsed() { return blockPoolUsed; } - public boolean addBlock(BlockInfo b) { + boolean addBlock(BlockInfo b) { if(!b.addStorage(this)) return false; // add to the head of the data-node list @@ -216,7 +210,7 @@ public class DatanodeStorageInfo { return true; } - public boolean removeBlock(BlockInfo b) { + boolean removeBlock(BlockInfo b) { blockList = b.listRemove(blockList, this); if (b.removeStorage(this)) { numBlocks--; @@ -226,7 +220,7 @@ public class DatanodeStorageInfo { } } - public int numBlocks() { + int numBlocks() { return numBlocks; } @@ -249,11 +243,11 @@ public class DatanodeStorageInfo { * @return the head of the blockList */ @VisibleForTesting - protected BlockInfo getHead(){ + BlockInfo getBlockListHeadForTesting(){ return blockList; } - public void updateState(StorageReport r) { + void updateState(StorageReport r) { capacity = r.getCapacity(); dfsUsed = r.getDfsUsed(); remaining = r.getRemaining(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java index 9a05c711fcf..6b07b789341 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java @@ -198,8 +198,8 @@ class PendingReplicationBlocks { } } - void decrementReplicas(DatanodeDescriptor target) { - targets.remove(target); + void decrementReplicas(DatanodeDescriptor dn) { + targets.remove(dn); } int getNumReplicas() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java index a1882c468d2..3d661f1b3a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java @@ -237,8 +237,7 @@ public abstract class Storage extends StorageInfo { final StorageDirType dirType; // storage dir type FileLock lock; // storage lock - //TODO HDFS-2832: Consider moving this out of StorageDirectory. - String storageUuid = null; // Storage directory identifier. + private String storageUuid = null; // Storage directory identifier. public StorageDirectory(File dir) { // default dirType is null diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index 858e0339c23..8eef1b6099a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -331,8 +331,7 @@ class BPOfferService { } } - synchronized DatanodeRegistration createRegistration() - throws IOException { + synchronized DatanodeRegistration createRegistration() throws IOException { Preconditions.checkState(bpNSInfo != null, "getRegistration() can only be called after initial handshake"); return dn.createBPRegistration(bpNSInfo); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 2612d1b9701..6f15c4a6be0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -251,10 +251,7 @@ class BPServiceActor implements Runnable { DatanodeInfo[] dnArr = { new DatanodeInfo(bpRegistration) }; String[] uuids = { storageUuid }; StorageType[] types = { storageType }; - // TODO: Corrupt flag is set to false for compatibility. We can probably - // set it to true here. - LocatedBlock[] blocks = { - new LocatedBlock(block, dnArr, uuids, types) }; + LocatedBlock[] blocks = { new LocatedBlock(block, dnArr, uuids, types) }; try { bpNamenode.reportBadBlocks(blocks); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index 20bb4596737..3106f7f688b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -255,8 +255,8 @@ class BlockReceiver implements Closeable { /** Return the datanode object. */ DataNode getDataNode() {return datanode;} - public Replica getReplicaInfo() { - return replicaInfo; + String getStorageUuid() { + return replicaInfo.getStorageUuid(); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 5a78ae2e501..6d61421bc91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -18,10 +18,40 @@ package org.apache.hadoop.hdfs.server.datanode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.protobuf.BlockingService; +import static org.apache.hadoop.hdfs.DFSConfigKeys.*; +import static org.apache.hadoop.util.ExitUtil.terminate; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.net.URI; +import java.net.UnknownHostException; +import java.nio.channels.ClosedByInterruptException; +import java.nio.channels.SocketChannel; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.management.ObjectName; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -38,15 +68,37 @@ import org.apache.hadoop.hdfs.HDFSPolicyProvider; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.net.DomainPeerServer; import org.apache.hadoop.hdfs.net.TcpPeerServer; -import org.apache.hadoop.hdfs.protocol.*; -import org.apache.hadoop.hdfs.protocol.datatransfer.*; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; +import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; +import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; +import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage; +import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferEncryptor; +import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol; +import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; +import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.ClientDatanodeProtocolService; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.DNTransferAckProto; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status; import org.apache.hadoop.hdfs.protocol.proto.InterDatanodeProtocolProtos.InterDatanodeProtocolService; -import org.apache.hadoop.hdfs.protocolPB.*; -import org.apache.hadoop.hdfs.security.token.block.*; +import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolPB; +import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolServerSideTranslatorPB; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; +import org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolPB; +import org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolServerSideTranslatorPB; +import org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB; +import org.apache.hadoop.hdfs.protocolPB.PBHelper; +import org.apache.hadoop.hdfs.security.token.block.BlockPoolTokenSecretManager; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode; +import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; +import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; @@ -59,8 +111,12 @@ import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; import org.apache.hadoop.hdfs.server.datanode.web.resources.DatanodeWebHdfsMethods; import org.apache.hadoop.hdfs.server.namenode.FileChecksumServlets; import org.apache.hadoop.hdfs.server.namenode.StreamFile; -import org.apache.hadoop.hdfs.server.protocol.*; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; +import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; +import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.Param; import org.apache.hadoop.http.HttpServer; @@ -82,23 +138,21 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.util.*; +import org.apache.hadoop.util.Daemon; +import org.apache.hadoop.util.DiskChecker; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; +import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.hadoop.util.JvmPauseMonitor; +import org.apache.hadoop.util.ServicePlugin; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.VersionInfo; import org.mortbay.util.ajax.JSON; -import java.io.*; -import java.net.*; -import java.nio.channels.ClosedByInterruptException; -import java.nio.channels.SocketChannel; -import java.security.PrivilegedExceptionAction; -import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; - -import javax.management.ObjectName; - -import static org.apache.hadoop.hdfs.DFSConfigKeys.*; -import static org.apache.hadoop.util.ExitUtil.terminate; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.protobuf.BlockingService; /********************************************************** * DataNode is a class (and program) that stores a set of @@ -204,7 +258,7 @@ public class DataNode extends Configured private JvmPauseMonitor pauseMonitor; private SecureResources secureResources = null; - private AbstractList dataDirs; + private List dataDirs; private Configuration conf; private final long maxNumberOfBlocksToLog; @@ -219,7 +273,7 @@ public class DataNode extends Configured * and a namenode proxy */ DataNode(final Configuration conf, - final AbstractList dataDirs, + final List dataDirs, final SecureResources resources) throws IOException { super(conf); this.maxNumberOfBlocksToLog = conf.getLong(DFS_MAX_NUM_BLOCKS_TO_LOG_KEY, @@ -640,7 +694,7 @@ public class DataNode extends Configured * @throws IOException */ void startDataNode(Configuration conf, - AbstractList dataDirs, + List dataDirs, // DatanodeProtocol namenode, SecureResources resources ) throws IOException { @@ -915,7 +969,7 @@ public class DataNode extends Configured /** * NB: The datanode can perform data transfer on the streaming * address however clients are given the IPC IP address for data - * transfer, and that may be a different address. + * transfer, and that may be a different address. * * @return socket address for data transfer */ @@ -1660,21 +1714,19 @@ public class DataNode extends Configured return makeInstance(dataLocations, conf, resources); } - static Collection parseStorageLocations( - Collection rawLocations) { + public static List getStorageLocations(Configuration conf) { + Collection rawLocations = + conf.getTrimmedStringCollection(DFS_DATANODE_DATA_DIR_KEY); List locations = new ArrayList(rawLocations.size()); for(String locationString : rawLocations) { - StorageLocation location; + final StorageLocation location; try { location = StorageLocation.parse(locationString); } catch (IOException ioe) { - LOG.error("Failed to parse storage location " + locationString); - continue; - } catch (IllegalArgumentException iae) { - LOG.error(iae.toString()); - continue; + throw new IllegalArgumentException("Failed to parse conf property " + + DFS_DATANODE_DATA_DIR_KEY + ": " + locationString, ioe); } locations.add(location); @@ -1683,12 +1735,6 @@ public class DataNode extends Configured return locations; } - public static Collection getStorageLocations( - Configuration conf) { - return parseStorageLocations( - conf.getTrimmedStringCollection(DFS_DATANODE_DATA_DIR_KEY)); - } - /** Instantiate & Start a single datanode daemon and wait for it to finish. * If this thread is specifically interrupted, it will stop waiting. */ @@ -1760,7 +1806,7 @@ public class DataNode extends Configured DFS_DATANODE_DATA_DIR_PERMISSION_DEFAULT)); DataNodeDiskChecker dataNodeDiskChecker = new DataNodeDiskChecker(permission); - ArrayList locations = + List locations = checkStorageLocations(dataDirs, localFS, dataNodeDiskChecker); DefaultMetricsSystem.initialize("DataNode"); @@ -1769,20 +1815,21 @@ public class DataNode extends Configured } // DataNode ctor expects AbstractList instead of List or Collection... - static ArrayList checkStorageLocations( + static List checkStorageLocations( Collection dataDirs, LocalFileSystem localFS, DataNodeDiskChecker dataNodeDiskChecker) throws IOException { ArrayList locations = new ArrayList(); StringBuilder invalidDirs = new StringBuilder(); for (StorageLocation location : dataDirs) { + final URI uri = location.getUri(); try { - dataNodeDiskChecker.checkDir(localFS, new Path(location.getUri())); + dataNodeDiskChecker.checkDir(localFS, new Path(uri)); locations.add(location); } catch (IOException ioe) { LOG.warn("Invalid " + DFS_DATANODE_DATA_DIR_KEY + " " + location.getFile() + " : ", ioe); - invalidDirs.append("\"").append(location.getUri().getPath()).append("\" "); + invalidDirs.append("\"").append(uri.getPath()).append("\" "); } } if (locations.size() == 0) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index e03151c64b1..70d3e21cd83 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -105,7 +105,7 @@ public class DataStorage extends Storage { /** Create an ID for this storage. */ public synchronized void createStorageID(StorageDirectory sd) { if (sd.getStorageUuid() == null) { - sd.setStorageUuid(DatanodeStorage.newStorageID()); + sd.setStorageUuid(DatanodeStorage.generateUuid()); } } @@ -215,8 +215,8 @@ public class DataStorage extends Storage { // Create list of storage directories for the block pool Collection bpDataDirs = new ArrayList(); - for(Iterator it = dataDirs.iterator(); it.hasNext();) { - File dnRoot = it.next().getFile(); + for(StorageLocation dir : dataDirs) { + File dnRoot = dir.getFile(); File bpRoot = BlockPoolSliceStorage.getBpRoot(bpID, new File(dnRoot, STORAGE_DIR_CURRENT)); bpDataDirs.add(bpRoot); @@ -269,7 +269,7 @@ public class DataStorage extends Storage { if (sd.getStorageUuid() == null) { // Assign a new Storage UUID. - sd.setStorageUuid(UUID.randomUUID().toString()); + sd.setStorageUuid(DatanodeStorage.generateUuid()); } writeProperties(sd); @@ -305,8 +305,7 @@ public class DataStorage extends Storage { /* * Read ClusterID, StorageID, StorageType, CTime from * DataStorage VERSION file and verify them. - * Always called just after reading the properties from the VERSION - * file. + * Always called just after reading the properties from the VERSION file. */ @Override protected void setFieldsFromProperties(Properties props, StorageDirectory sd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index 24345841211..522d9e2ef4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -447,7 +447,7 @@ class DataXceiver extends Receiver implements Runnable { String mirrorNode = null; // the name:port of next target String firstBadLink = ""; // first datanode that failed in connection setup Status mirrorInStatus = SUCCESS; - Replica replica; + final String storageUuid; try { if (isDatanode || stage != BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) { @@ -458,10 +458,10 @@ class DataXceiver extends Receiver implements Runnable { stage, latestGenerationStamp, minBytesRcvd, maxBytesRcvd, clientname, srcDataNode, datanode, requestedChecksum, cachingStrategy); - replica = blockReceiver.getReplicaInfo(); + storageUuid = blockReceiver.getStorageUuid(); } else { - replica = - datanode.data.recoverClose(block, latestGenerationStamp, minBytesRcvd); + storageUuid = datanode.data.recoverClose( + block, latestGenerationStamp, minBytesRcvd); } // @@ -593,8 +593,7 @@ class DataXceiver extends Receiver implements Runnable { // the block is finalized in the PacketResponder. if (isDatanode || stage == BlockConstructionStage.PIPELINE_CLOSE_RECOVERY) { - datanode.closeBlock( - block, DataNode.EMPTY_DEL_HINT, replica.getStorageUuid()); + datanode.closeBlock(block, DataNode.EMPTY_DEL_HINT, storageUuid); LOG.info("Received " + block + " src: " + remoteAddress + " dest: " + localAddress + " of size " + block.getNumBytes()); } @@ -864,7 +863,7 @@ class DataXceiver extends Receiver implements Runnable { // notify name node datanode.notifyNamenodeReceivedBlock( - block, delHint, blockReceiver.getReplicaInfo().getStorageUuid()); + block, delHint, blockReceiver.getStorageUuid()); LOG.info("Moved " + block + " from " + peer.getRemoteAddressString() + ", delHint=" + delHint); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java index 17ec35d6fb6..9ebd5740efd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java index 1a852c34668..90658733ecd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java @@ -61,10 +61,6 @@ public class FinalizedReplica extends ReplicaInfo { this.unlinked = from.isUnlinked(); } - public FinalizedReplica(ReplicaInfo replicaInfo) { - super(replicaInfo); - } - @Override // ReplicaInfo public ReplicaState getState() { return ReplicaState.FINALIZED; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java index bc1e57a0038..66ab18f31fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java @@ -18,18 +18,16 @@ package org.apache.hadoop.hdfs.server.datanode; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.StorageType; -import org.apache.hadoop.hdfs.server.common.Util; +import java.util.regex.Pattern; import java.io.File; import java.io.IOException; import java.net.URI; import java.util.regex.Matcher; -import static java.util.regex.Pattern.compile; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.StorageType; +import org.apache.hadoop.hdfs.server.common.Util; /** * Encapsulates the URI and storage medium that together describe a @@ -39,19 +37,15 @@ import static java.util.regex.Pattern.compile; */ @InterfaceAudience.Private public class StorageLocation { - public static final Log LOG = LogFactory.getLog(StorageLocation.class); final StorageType storageType; final File file; - // Regular expression that describes a storage uri with a storage type. - // e.g. [Disk]/storages/storage1/ - private static final String rawStringRegex = "^\\[(\\w*)\\](.+)$"; + /** Regular expression that describes a storage uri with a storage type. + * e.g. [Disk]/storages/storage1/ + */ + private static final Pattern regex = Pattern.compile("^\\[(\\w*)\\](.+)$"); - StorageLocation(URI uri) { - this(StorageType.DISK, uri); - } - - StorageLocation(StorageType storageType, URI uri) { + private StorageLocation(StorageType storageType, URI uri) { this.storageType = storageType; if (uri.getScheme() == null || @@ -59,8 +53,7 @@ public class StorageLocation { // drop any (illegal) authority in the URI for backwards compatibility this.file = new File(uri.getPath()); } else { - throw new IllegalArgumentException( - "Got an Unsupported URI schema in " + uri + ". Ignoring ..."); + throw new IllegalArgumentException("Unsupported URI schema in " + uri); } } @@ -68,7 +61,7 @@ public class StorageLocation { return this.storageType; } - public URI getUri() { + URI getUri() { return file.toURI(); } @@ -85,29 +78,24 @@ public class StorageLocation { * @return A StorageLocation object if successfully parsed, null otherwise. * Does not throw any exceptions. */ - public static StorageLocation parse(String rawLocation) throws IOException { - Matcher matcher = compile(rawStringRegex).matcher(rawLocation); - StorageType storageType = StorageType.DISK; + static StorageLocation parse(String rawLocation) throws IOException { + Matcher matcher = regex.matcher(rawLocation); + StorageType storageType = StorageType.DEFAULT; String location = rawLocation; if (matcher.matches()) { String classString = matcher.group(1); location = matcher.group(2); if (!classString.isEmpty()) { - try { - storageType = StorageType.valueOf(classString.toUpperCase()); - } catch (RuntimeException re) { - LOG.error("Unable to parse storage type: " + re.toString() + - ". Using the default storage type for directory " + - location); - } + storageType = StorageType.valueOf(classString.toUpperCase()); } } return new StorageLocation(storageType, Util.stringAsURI(location)); } + @Override public String toString() { - return "[" + storageType.toString() + "]" + file.toURI().toString(); + return "[" + storageType + "]" + file.toURI(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index c870d6a1a59..b9d6c5bd14a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@ -242,9 +242,10 @@ public interface FsDatasetSpi extends FSDatasetMBean { * @param b block * @param newGS the new generation stamp for the replica * @param expectedBlockLen the number of bytes the replica is expected to have + * @return the storage uuid of the replica. * @throws IOException */ - public Replica recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen + public String recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen ) throws IOException; /** @@ -264,15 +265,6 @@ public interface FsDatasetSpi extends FSDatasetMBean { */ public void unfinalizeBlock(ExtendedBlock b) throws IOException; - /** - * TODO HDFS-2832: Deprecate this when we fix tests. - * Returns the block report - the full list of blocks stored under a - * block pool - * @param bpid Block Pool Id - * @return - the block report - the full list of blocks stored - */ - public BlockListAsLongs getBlockReport(String bpid); - /** * Returns one block report per volume. * @param bpid Block Pool Id diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 0900a3ec143..ac7024481a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -197,8 +197,8 @@ class FsDatasetImpl implements FsDatasetSpi { private final int validVolsRequired; // TODO HDFS-2832: Consider removing duplicated block info from these - // two maps. This might require some refactoring - // rewrite of FsDatasetImpl. + // two maps and move the perVolumeReplicaMap to FsVolumeImpl. + // This might require some refactoring. final ReplicaMap volumeMap; // Map from StorageID to ReplicaMap. @@ -726,7 +726,7 @@ class FsDatasetImpl implements FsDatasetSpi { } @Override // FsDatasetSpi - public Replica recoverClose(ExtendedBlock b, long newGS, + public String recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException { LOG.info("Recover failed close " + b); // check replica's state @@ -737,7 +737,7 @@ class FsDatasetImpl implements FsDatasetSpi { if (replicaInfo.getState() == ReplicaState.RBW) { finalizeReplica(b.getBlockPoolId(), replicaInfo); } - return replicaInfo; + return replicaInfo.getStorageUuid(); } /** @@ -1083,14 +1083,6 @@ class FsDatasetImpl implements FsDatasetSpi { return cacheManager.getCachedBlocks(bpid); } - /** - * Generates a block report from the in-memory block map. - */ - @Override // FsDatasetSpi - public BlockListAsLongs getBlockReport(String bpid) { - return getBlockReportWithReplicaMap(bpid, volumeMap); - } - @Override public Map getBlockReports(String bpid) { Map blockReportMap = @@ -1114,7 +1106,7 @@ class FsDatasetImpl implements FsDatasetSpi { new ArrayList(volumeMap.size(bpid)); for (ReplicaInfo b : volumeMap.replicas(bpid)) { if(b.getState() == ReplicaState.FINALIZED) { - finalized.add(new FinalizedReplica(b)); + finalized.add(new FinalizedReplica((FinalizedReplica)b)); } } return finalized; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index 671996718be..708ffaf06e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -53,7 +53,7 @@ class FsVolumeList { * @param blockSize free space needed on the volume * @return next volume to store the block in. */ - // TODO this will be replaced by getting volume from StorageID directly later. + // TODO should choose volume with storage type synchronized FsVolumeImpl getNextVolume(long blockSize) throws IOException { return blockChooser.chooseVolume(volumes, blockSize); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java index d59c38644ff..fe478cfaee9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReplicaMap.java @@ -119,9 +119,7 @@ class ReplicaMap { } /** - * Add all entries from the given replica map into the local replica - * map. - * @param + * Add all entries from the given replica map into the local replica map. */ void addAll(ReplicaMap other) { map.putAll(other.map); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java index 644b294f506..da80f21e985 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java @@ -36,8 +36,7 @@ public class DatanodeStorage { private final StorageType storageType; /** - * Create a storage with {@link State#NORMAL} and - * {@link org.apache.hadoop.hdfs.StorageType#DEFAULT}. + * Create a storage with {@link State#NORMAL} and {@link StorageType#DEFAULT}. * * @param storageID */ @@ -45,10 +44,6 @@ public class DatanodeStorage { this(storageID, State.NORMAL, StorageType.DEFAULT); } - public DatanodeStorage(String sid, State s) { - this(sid, s, StorageType.DEFAULT); - } - public DatanodeStorage(String sid, State s, StorageType sm) { this.storageID = sid; this.state = s; @@ -69,11 +64,11 @@ public class DatanodeStorage { /** * Generate new storage ID. The format of this string can be changed - * in the future without requiring that old SotrageIDs be updated. + * in the future without requiring that old storage IDs be updated. * * @return unique storage ID */ - public static String newStorageID() { + public static String generateUuid() { return "DS-" + UUID.randomUUID(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto index f6ed6b09de1..236a7350705 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto @@ -55,7 +55,7 @@ message DatanodeStorageProto { required string storageUuid = 1; optional StorageState state = 2 [default = NORMAL]; - optional StorageTypeProto storageType = 3; + optional StorageTypeProto storageType = 3 [default = DISK]; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index f121d127ee9..ad452f97330 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -853,8 +853,8 @@ public class DFSTestUtil { public static DatanodeStorageInfo createDatanodeStorageInfo( String storageID, String ip, String rack) { final DatanodeStorage storage = new DatanodeStorage(storageID); - return new DatanodeStorageInfo( - BlockManagerTestUtil.getDatanodeDescriptor(ip, rack, storage), storage); + final DatanodeDescriptor dn = BlockManagerTestUtil.getDatanodeDescriptor(ip, rack, storage); + return BlockManagerTestUtil.newDatanodeStorageInfo(dn, storage); } public static DatanodeDescriptor[] toDatanodeDescriptor( DatanodeStorageInfo[] storages) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 98afa7aa79f..fa891ec4bac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -55,6 +55,7 @@ import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Random; import org.apache.commons.logging.Log; @@ -1969,12 +1970,12 @@ public class MiniDFSCluster { * @param dataNodeIndex - data node whose block report is desired - the index is same as for getDataNodes() * @return the block report for the specified data node */ - public Iterable getBlockReport(String bpid, int dataNodeIndex) { + public Map getBlockReport(String bpid, int dataNodeIndex) { if (dataNodeIndex < 0 || dataNodeIndex > dataNodes.size()) { throw new IndexOutOfBoundsException(); } final DataNode dn = dataNodes.get(dataNodeIndex).datanode; - return DataNodeTestUtils.getFSDataset(dn).getBlockReport(bpid); + return DataNodeTestUtils.getFSDataset(dn).getBlockReports(bpid); } @@ -1983,11 +1984,12 @@ public class MiniDFSCluster { * @return block reports from all data nodes * BlockListAsLongs is indexed in the same order as the list of datanodes returned by getDataNodes() */ - public Iterable[] getAllBlockReports(String bpid) { + public List> getAllBlockReports(String bpid) { int numDataNodes = dataNodes.size(); - Iterable[] result = new BlockListAsLongs[numDataNodes]; + final List> result + = new ArrayList>(numDataNodes); for (int i = 0; i < numDataNodes; ++i) { - result[i] = getBlockReport(bpid, i); + result.add(getBlockReport(bpid, i)); } return result; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index 63bf317a5a3..f065dc923af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -23,6 +23,7 @@ import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.Scanner; import java.util.concurrent.atomic.AtomicInteger; @@ -35,6 +36,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.tools.DFSAdmin; @@ -1392,11 +1394,14 @@ public class TestDFSShell { List files = new ArrayList(); List datanodes = cluster.getDataNodes(); String poolId = cluster.getNamesystem().getBlockPoolId(); - Iterable[] blocks = cluster.getAllBlockReports(poolId); - for(int i = 0; i < blocks.length; i++) { + List> blocks = cluster.getAllBlockReports(poolId); + for(int i = 0; i < blocks.size(); i++) { DataNode dn = datanodes.get(i); - for(Block b : blocks[i]) { - files.add(DataNodeTestUtils.getFile(dn, poolId, b.getBlockId())); + Map map = blocks.get(i); + for(Map.Entry e : map.entrySet()) { + for(Block b : e.getValue()) { + files.add(DataNodeTestUtils.getFile(dn, poolId, b.getBlockId())); + } } } return files; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java index a42b0344418..da2b8bbfe80 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java @@ -22,16 +22,16 @@ import static org.junit.Assert.assertEquals; import java.io.IOException; import java.net.InetSocketAddress; import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -136,7 +136,7 @@ public class TestInjectionForSimulatedStorage { DFSTestUtil.createFile(cluster.getFileSystem(), testPath, filesize, filesize, blockSize, (short) numDataNodes, 0L); waitForBlockReplication(testFile, dfsClient.getNamenode(), numDataNodes, 20); - Iterable[] blocksList = cluster.getAllBlockReports(bpid); + List> blocksList = cluster.getAllBlockReports(bpid); cluster.shutdown(); cluster = null; @@ -157,9 +157,11 @@ public class TestInjectionForSimulatedStorage { .build(); cluster.waitActive(); Set uniqueBlocks = new HashSet(); - for (int i=0; i map : blocksList) { + for(BlockListAsLongs blockList : map.values()) { + for(Block b : blockList) { + uniqueBlocks.add(new Block(b)); + } } } // Insert all the blocks in the first data node diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index 60c6d6bbbb3..4c02da18d85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -432,17 +432,18 @@ public class TestPBHelper { DFSTestUtil.getLocalDatanodeInfo("127.0.0.1", "h3", AdminStates.NORMAL) }; + String[] storageIDs = {"s1", "s2", "s3"}; StorageType[] media = { StorageType.DISK, StorageType.SSD, StorageType.DISK }; LocatedBlock lb = new LocatedBlock( - new ExtendedBlock("bp12", 12345, 10, 53), dnInfos, 5, false); + new ExtendedBlock("bp12", 12345, 10, 53), + dnInfos, storageIDs, media, 5, false, new DatanodeInfo[]{}); lb.setBlockToken(new Token( "identifier".getBytes(), "password".getBytes(), new Text("kind"), new Text("service"))); - lb.setStorageTypes(media); return lb; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java index 41dbbcf5012..cfc9750967b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java @@ -23,7 +23,6 @@ import java.util.Collection; import java.util.HashSet; import java.util.Set; -import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.protocol.Block; @@ -219,11 +218,16 @@ public class BlockManagerTestUtil { bm.getDatanodeManager().getHeartbeatManager().heartbeatCheck(); } + public static DatanodeStorageInfo updateStorage(DatanodeDescriptor dn, + DatanodeStorage s) { + return dn.updateStorage(s); + } + public static DatanodeDescriptor getLocalDatanodeDescriptor( boolean initializeStorage) { DatanodeDescriptor dn = new DatanodeDescriptor(DFSTestUtil.getLocalDatanodeID()); if (initializeStorage) { - dn.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); + dn.updateStorage(new DatanodeStorage(DatanodeStorage.generateUuid())); } return dn; } @@ -231,7 +235,7 @@ public class BlockManagerTestUtil { public static DatanodeDescriptor getDatanodeDescriptor(String ipAddr, String rackLocation, boolean initializeStorage) { return getDatanodeDescriptor(ipAddr, rackLocation, - initializeStorage? new DatanodeStorage(DatanodeStorage.newStorageID()): null); + initializeStorage? new DatanodeStorage(DatanodeStorage.generateUuid()): null); } public static DatanodeDescriptor getDatanodeDescriptor(String ipAddr, @@ -244,6 +248,10 @@ public class BlockManagerTestUtil { return dn; } + public static DatanodeStorageInfo newDatanodeStorageInfo( + DatanodeDescriptor dn, DatanodeStorage s) { + return new DatanodeStorageInfo(dn, s); + } public static StorageReport[] getStorageReportsForDatanode( DatanodeDescriptor dnd) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java index 79e0c8573d4..78a77c44601 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java @@ -76,7 +76,7 @@ public class TestBlockInfo { } assertEquals("There should be MAX_BLOCK blockInfo's", MAX_BLOCKS, len); - headIndex = dd.getHead().findStorageInfo(dd); + headIndex = dd.getBlockListHeadForTesting().findStorageInfo(dd); LOG.info("Moving each block to the head of the list..."); for (int i = 0; i < MAX_BLOCKS; i++) { @@ -84,23 +84,23 @@ public class TestBlockInfo { headIndex = dd.moveBlockToHead(blockInfoList.get(i), curIndex, headIndex); // the moved element must be at the head of the list assertEquals("Block should be at the head of the list now.", - blockInfoList.get(i), dd.getHead()); + blockInfoList.get(i), dd.getBlockListHeadForTesting()); } // move head of the list to the head - this should not change the list LOG.info("Moving head to the head..."); - BlockInfo temp = dd.getHead(); + BlockInfo temp = dd.getBlockListHeadForTesting(); curIndex = 0; headIndex = 0; dd.moveBlockToHead(temp, curIndex, headIndex); assertEquals( "Moving head to the head of the list shopuld not change the list", - temp, dd.getHead()); + temp, dd.getBlockListHeadForTesting()); // check all elements of the list against the original blockInfoList LOG.info("Checking elements of the list..."); - temp = dd.getHead(); + temp = dd.getBlockListHeadForTesting(); assertNotNull("Head should not be null", temp); int c = MAX_BLOCKS - 1; while (temp != null) { @@ -110,7 +110,7 @@ public class TestBlockInfo { } LOG.info("Moving random blocks to the head of the list..."); - headIndex = dd.getHead().findStorageInfo(dd); + headIndex = dd.getBlockListHeadForTesting().findStorageInfo(dd); Random rand = new Random(); for (int i = 0; i < MAX_BLOCKS; i++) { int j = rand.nextInt(MAX_BLOCKS); @@ -118,7 +118,7 @@ public class TestBlockInfo { headIndex = dd.moveBlockToHead(blockInfoList.get(j), curIndex, headIndex); // the moved element must be at the head of the list assertEquals("Block should be at the head of the list now.", - blockInfoList.get(j), dd.getHead()); + blockInfoList.get(j), dd.getBlockListHeadForTesting()); } } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 38e3cc52dc9..e632ed1ca97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -103,7 +103,7 @@ public class TestBlockManager { // construct network topology for (DatanodeDescriptor dn : nodesToAdd) { cluster.add(dn); - dn.getStorageInfos()[0].setUtilization( + dn.getStorageInfos()[0].setUtilizationForTesting( 2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L); dn.updateHeartbeat( @@ -295,7 +295,7 @@ public class TestBlockManager { // the third off-rack replica. DatanodeDescriptor rackCNode = DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/rackC"); - rackCNode.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); + rackCNode.updateStorage(new DatanodeStorage(DatanodeStorage.generateUuid())); addNodes(ImmutableList.of(rackCNode)); try { DatanodeStorageInfo[] pipeline2 = scheduleSingleReplication(blockInfo); @@ -531,7 +531,7 @@ public class TestBlockManager { DatanodeStorageInfo ds = node.getStorageInfos()[0]; // TODO: Needs to be fixed. DatanodeUuid is not storageID. - node.setDatanodeUuid(ds.getStorageID()); + node.setDatanodeUuidForTesting(ds.getStorageID()); node.isAlive = true; @@ -577,7 +577,7 @@ public class TestBlockManager { DatanodeStorageInfo ds = node.getStorageInfos()[0]; // TODO: Needs to be fixed. DatanodeUuid is not storageID. - node.setDatanodeUuid(ds.getStorageID()); + node.setDatanodeUuidForTesting(ds.getStorageID()); node.isAlive = true; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java index a5956db8a0c..4b27aa68b3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java @@ -63,7 +63,7 @@ public class TestHeartbeatHandling { final DatanodeRegistration nodeReg = DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId); final DatanodeDescriptor dd = NameNodeAdapter.getDatanode(namesystem, nodeReg); - final String storageID = DatanodeStorage.newStorageID(); + final String storageID = DatanodeStorage.generateUuid(); dd.updateStorage(new DatanodeStorage(storageID)); final int REMAINING_BLOCKS = 1; @@ -146,15 +146,15 @@ public class TestHeartbeatHandling { final DatanodeRegistration nodeReg1 = DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId); final DatanodeDescriptor dd1 = NameNodeAdapter.getDatanode(namesystem, nodeReg1); - dd1.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); + dd1.updateStorage(new DatanodeStorage(DatanodeStorage.generateUuid())); final DatanodeRegistration nodeReg2 = DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(1), poolId); final DatanodeDescriptor dd2 = NameNodeAdapter.getDatanode(namesystem, nodeReg2); - dd2.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); + dd2.updateStorage(new DatanodeStorage(DatanodeStorage.generateUuid())); final DatanodeRegistration nodeReg3 = DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(2), poolId); final DatanodeDescriptor dd3 = NameNodeAdapter.getDatanode(namesystem, nodeReg3); - dd3.updateStorage(new DatanodeStorage(DatanodeStorage.newStorageID())); + dd3.updateStorage(new DatanodeStorage(DatanodeStorage.generateUuid())); try { namesystem.writeLock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java index 01368d3f999..1c3f75a68d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java @@ -103,7 +103,7 @@ public class TestOverReplicatedBlocks { String corruptMachineName = corruptDataNode.getXferAddr(); for (DatanodeDescriptor datanode : hm.getDatanodes()) { if (!corruptMachineName.equals(datanode.getXferAddr())) { - datanode.getStorageInfos()[0].setUtilization(100L, 100L, 0, 100L); + datanode.getStorageInfos()[0].setUtilizationForTesting(100L, 100L, 0, 100L); datanode.updateHeartbeat( BlockManagerTestUtil.getStorageReportsForDatanode(datanode), 0L, 0L, 0, 0); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index 7bad49f1f83..fcff8cce026 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -93,7 +93,7 @@ public class TestReplicationPolicy { private static void updateHeartbeatWithUsage(DatanodeDescriptor dn, long capacity, long dfsUsed, long remaining, long blockPoolUsed, long dnCacheCapacity, long dnCacheUsed, int xceiverCount, int volFailures) { - dn.getStorageInfos()[0].setUtilization( + dn.getStorageInfos()[0].setUtilizationForTesting( capacity, dfsUsed, remaining, blockPoolUsed); dn.updateHeartbeat( BlockManagerTestUtil.getStorageReportsForDatanode(dn), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java index 855f70e2976..23209f0e28b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java @@ -150,7 +150,7 @@ public class TestReplicationPolicyWithNodeGroup { long capacity, long dfsUsed, long remaining, long blockPoolUsed, long dnCacheCapacity, long dnCacheUsed, int xceiverCount, int volFailures) { - dn.getStorageInfos()[0].setUtilization( + dn.getStorageInfos()[0].setUtilizationForTesting( capacity, dfsUsed, remaining, blockPoolUsed); dn.updateHeartbeat( BlockManagerTestUtil.getStorageReportsForDatanode(dn), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java index 93b64bf0009..37d594eef53 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java @@ -17,12 +17,33 @@ */ package org.apache.hadoop.hdfs.server.common; -import com.google.common.base.Strings; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.StringReader; +import java.net.InetSocketAddress; +import java.text.MessageFormat; +import java.util.ArrayList; + +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.jsp.JspWriter; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; @@ -48,20 +69,7 @@ import org.mockito.stubbing.Answer; import org.xml.sax.InputSource; import org.xml.sax.SAXException; -import javax.servlet.ServletContext; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.jsp.JspWriter; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import java.io.IOException; -import java.io.StringReader; -import java.net.InetSocketAddress; -import java.text.MessageFormat; -import java.util.ArrayList; - -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; +import com.google.common.base.Strings; public class TestJspHelper { @@ -459,8 +467,8 @@ public class TestJspHelper { DatanodeDescriptor dnDesc2 = new DatanodeDescriptor(dnId2, "rack2"); // Update the DatanodeDescriptors with their attached storages. - dnDesc1.updateStorage(new DatanodeStorage("dnStorage1")); - dnDesc2.updateStorage(new DatanodeStorage("dnStorage2")); + BlockManagerTestUtil.updateStorage(dnDesc1, new DatanodeStorage("dnStorage1")); + BlockManagerTestUtil.updateStorage(dnDesc2, new DatanodeStorage("dnStorage2")); StorageReport[] report1 = new StorageReport[] { new StorageReport("dnStorage1", false, 1024, 100, 924, 100) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 56e22feae5f..7693c307534 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -324,7 +324,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { private static class SimulatedStorage { private Map map = new HashMap(); - private final String storageUuid = "SimulatedStroage-" + DatanodeStorage.newStorageID(); + private final String storageUuid = "SimulatedStroage-" + DatanodeStorage.generateUuid(); private final long capacity; // in bytes @@ -470,8 +470,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { } } - @Override - public synchronized BlockListAsLongs getBlockReport(String bpid) { + synchronized BlockListAsLongs getBlockReport(String bpid) { final List blocks = new ArrayList(); final Map map = blockMap.get(bpid); if (map != null) { @@ -684,7 +683,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { } @Override // FsDatasetSpi - public Replica recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen) + public String recoverClose(ExtendedBlock b, long newGS, long expectedBlockLen) throws IOException { final Map map = getMap(b.getBlockPoolId()); BInfo binfo = map.get(b.getLocalBlock()); @@ -698,7 +697,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { map.remove(b.getLocalBlock()); binfo.theBlock.setGenerationStamp(newGS); map.put(binfo.theBlock, binfo); - return binfo; + return binfo.getStorageUuid(); } @Override // FsDatasetSpi diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java index 53cf5db42e4..c4b7e8f6f37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java @@ -35,7 +35,6 @@ import static org.mockito.Mockito.when; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; -import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; @@ -46,9 +45,18 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; -import org.apache.hadoop.hdfs.*; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -73,7 +81,10 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.DataChecksum; import org.apache.log4j.Level; -import org.junit.*; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -125,7 +136,7 @@ public class TestBlockRecovery { File dataDir = new File(DATA_DIR); FileUtil.fullyDelete(dataDir); dataDir.mkdirs(); - StorageLocation location = new StorageLocation(new URI(dataDir.getPath())); + StorageLocation location = StorageLocation.parse(dataDir.getPath()); locations.add(location); final DatanodeProtocolClientSideTranslatorPB namenode = mock(DatanodeProtocolClientSideTranslatorPB.class); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java index 57ee14464cc..53babb471db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.datanode; import java.io.*; -import java.net.URI; import java.util.*; import org.apache.hadoop.conf.Configuration; @@ -40,7 +39,7 @@ public class TestDataDirs { @Test (timeout = 30000) public void testDataDirParsing() throws Throwable { Configuration conf = new Configuration(); - ArrayList locations; + List locations; File dir0 = new File("/dir0"); File dir1 = new File("/dir1"); File dir2 = new File("/dir2"); @@ -50,7 +49,7 @@ public class TestDataDirs { // type is not case-sensitive String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3"; conf.set(DFS_DATANODE_DATA_DIR_KEY, locations1); - locations = new ArrayList(DataNode.getStorageLocations(conf)); + locations = DataNode.getStorageLocations(conf); assertThat(locations.size(), is(4)); assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); assertThat(locations.get(0).getUri(), is(dir0.toURI())); @@ -61,23 +60,21 @@ public class TestDataDirs { assertThat(locations.get(3).getStorageType(), is(StorageType.DISK)); assertThat(locations.get(3).getUri(), is(dir3.toURI())); - // Verify that an unrecognized storage type is ignored. + // Verify that an unrecognized storage type result in an exception. String locations2 = "[BadMediaType]/dir0,[ssd]/dir1,[disk]/dir2"; conf.set(DFS_DATANODE_DATA_DIR_KEY, locations2); - locations = new ArrayList(DataNode.getStorageLocations(conf)); - assertThat(locations.size(), is(3)); - assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); - assertThat(locations.get(0).getUri(), is(dir0.toURI())); - assertThat(locations.get(1).getStorageType(), is(StorageType.SSD)); - assertThat(locations.get(1).getUri(), is(dir1.toURI())); - assertThat(locations.get(2).getStorageType(), is(StorageType.DISK)); - assertThat(locations.get(2).getUri(), is(dir2.toURI())); + try { + locations = DataNode.getStorageLocations(conf); + fail(); + } catch(IllegalArgumentException iae) { + DataNode.LOG.info("The exception is expected.", iae); + } // Assert that a string with no storage type specified is // correctly parsed and the default storage type is picked up. String locations3 = "/dir0,/dir1"; conf.set(DFS_DATANODE_DATA_DIR_KEY, locations3); - locations = new ArrayList(DataNode.getStorageLocations(conf)); + locations = DataNode.getStorageLocations(conf); assertThat(locations.size(), is(2)); assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); assertThat(locations.get(0).getUri(), is(dir0.toURI())); @@ -94,11 +91,11 @@ public class TestDataDirs { LocalFileSystem fs = mock(LocalFileSystem.class); AbstractList locations = new ArrayList(); - locations.add(new StorageLocation(new URI("file:/p1/"))); - locations.add(new StorageLocation(new URI("file:/p2/"))); - locations.add(new StorageLocation(new URI("file:/p3/"))); + locations.add(StorageLocation.parse("file:/p1/")); + locations.add(StorageLocation.parse("file:/p2/")); + locations.add(StorageLocation.parse("file:/p3/")); - ArrayList checkedLocations = + List checkedLocations = DataNode.checkStorageLocations(locations, fs, diskChecker); assertEquals("number of valid data dirs", 1, checkedLocations.size()); String validDir = checkedLocations.iterator().next().getFile().getPath(); From 81211123d8c42c1dfd164f3c75d20f117dfc3168 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sat, 23 Nov 2013 23:27:13 +0000 Subject: [PATCH 50/76] HDFS-5559. Fix TestDatanodeConfig in HDFS-2832. (Contributed by szetszwo) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1544891 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 ++ .../test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index beced410222..cb2b7bd01dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -118,3 +118,5 @@ IMPROVEMENTS: HDFS-5542. Fix TODO and clean up the code in HDFS-2832. (Contributed by szetszwo) + HDFS-5559. Fix TestDatanodeConfig in HDFS-2832. (Contributed by szetszwo) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java index 35b492ebb25..9a331069135 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs; import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.junit.Assume.assumeTrue; import java.io.File; @@ -82,7 +83,8 @@ public class TestDatanodeConfig { DataNode dn = null; try { dn = DataNode.createDataNode(new String[]{}, conf); - } catch(IOException e) { + fail(); + } catch(Exception e) { // expecting exception here } if(dn != null) From a1aa1836fb6831c25efe326cdfdc014370cf5957 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 3 Dec 2013 16:30:29 +0000 Subject: [PATCH 51/76] HDFS-5484. StorageType and State in DatanodeStorageInfo in NameNode is not accurate. (Contributed by Eric Sirianni) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1547462 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt | 3 +++ .../hadoop/hdfs/server/datanode/BPServiceActor.java | 8 +++----- .../hdfs/server/datanode/fsdataset/FsDatasetSpi.java | 5 +++-- .../server/datanode/fsdataset/impl/FsDatasetImpl.java | 9 +++++---- .../server/datanode/fsdataset/impl/FsVolumeImpl.java | 6 ++++++ .../test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java | 9 +++++---- .../test/java/org/apache/hadoop/hdfs/TestDFSShell.java | 7 ++++--- .../hadoop/hdfs/TestInjectionForSimulatedStorage.java | 5 +++-- .../hadoop/hdfs/server/datanode/SimulatedFSDataset.java | 8 +++----- .../hadoop/hdfs/server/datanode/TestBlockReport.java | 8 +++----- .../hdfs/server/datanode/TestDataNodeVolumeFailure.java | 7 +++---- 11 files changed, 41 insertions(+), 34 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index cb2b7bd01dc..f822f9cf33f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -120,3 +120,6 @@ IMPROVEMENTS: HDFS-5559. Fix TestDatanodeConfig in HDFS-2832. (Contributed by szetszwo) + HDFS-5484. StorageType and State in DatanodeStorageInfo in NameNode is + not accurate. (Eric Sirianni via Arpit Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 6f15c4a6be0..c9317b35db4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -454,7 +454,7 @@ class BPServiceActor implements Runnable { long brCreateStartTime = now(); long totalBlockCount = 0; - Map perVolumeBlockLists = + Map perVolumeBlockLists = dn.getFSDataset().getBlockReports(bpos.getBlockPoolId()); // Send block report @@ -463,13 +463,11 @@ class BPServiceActor implements Runnable { new StorageBlockReport[perVolumeBlockLists.size()]; int i = 0; - for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { - String storageID = kvPair.getKey(); + for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { + DatanodeStorage dnStorage = kvPair.getKey(); BlockListAsLongs blockList = kvPair.getValue(); totalBlockCount += blockList.getNumberOfBlocks(); - // Dummy DatanodeStorage object just for sending the block report. - DatanodeStorage dnStorage = new DatanodeStorage(storageID); reports[i++] = new StorageBlockReport( dnStorage, blockList.getBlockListAsLongs()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index b9d6c5bd14a..415c6a985ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory; import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.util.DiskChecker.DiskErrorException; @@ -268,9 +269,9 @@ public interface FsDatasetSpi extends FSDatasetMBean { /** * Returns one block report per volume. * @param bpid Block Pool Id - * @return - a map of StorageID to block report for the volume. + * @return - a map of DatanodeStorage to block report for the volume. */ - public Map getBlockReports(String bpid); + public Map getBlockReports(String bpid); /** * Returns the cache report - the full list of cached block IDs of a diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 79c30738bd8..806e9f70402 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosing import org.apache.hadoop.hdfs.server.datanode.fsdataset.VolumeChoosingPolicy; import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.io.nativeio.NativeIO; @@ -1089,14 +1090,14 @@ class FsDatasetImpl implements FsDatasetSpi { } @Override - public Map getBlockReports(String bpid) { - Map blockReportMap = - new HashMap(); + public Map getBlockReports(String bpid) { + Map blockReportMap = + new HashMap(); for (FsVolumeImpl v : getVolumes()) { ReplicaMap rMap = perVolumeReplicaMap.get(v.getStorageID()); BlockListAsLongs blockList = getBlockReportWithReplicaMap(bpid, rMap); - blockReportMap.put(v.getStorageID(), blockList); + blockReportMap.put(v.toDatanodeStorage(), blockList); } return blockReportMap; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index b7e2ccd10db..9e5b0ebee4f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.datanode.DataStorage; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -332,5 +333,10 @@ class FsVolumeImpl implements FsVolumeSpi { public StorageType getStorageType() { return storageType; } + + DatanodeStorage toDatanodeStorage() { + return new DatanodeStorage(storageID, DatanodeStorage.State.NORMAL, storageType); + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index fa891ec4bac..abec6f01518 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -88,6 +88,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.hadoop.hdfs.web.HftpFileSystem; @@ -1970,7 +1971,7 @@ public class MiniDFSCluster { * @param dataNodeIndex - data node whose block report is desired - the index is same as for getDataNodes() * @return the block report for the specified data node */ - public Map getBlockReport(String bpid, int dataNodeIndex) { + public Map getBlockReport(String bpid, int dataNodeIndex) { if (dataNodeIndex < 0 || dataNodeIndex > dataNodes.size()) { throw new IndexOutOfBoundsException(); } @@ -1984,10 +1985,10 @@ public class MiniDFSCluster { * @return block reports from all data nodes * BlockListAsLongs is indexed in the same order as the list of datanodes returned by getDataNodes() */ - public List> getAllBlockReports(String bpid) { + public List> getAllBlockReports(String bpid) { int numDataNodes = dataNodes.size(); - final List> result - = new ArrayList>(numDataNodes); + final List> result + = new ArrayList>(numDataNodes); for (int i = 0; i < numDataNodes; ++i) { result.add(getBlockReport(bpid, i)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index f065dc923af..dd453644a0f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.tools.DFSAdmin; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; @@ -1394,11 +1395,11 @@ public class TestDFSShell { List files = new ArrayList(); List datanodes = cluster.getDataNodes(); String poolId = cluster.getNamesystem().getBlockPoolId(); - List> blocks = cluster.getAllBlockReports(poolId); + List> blocks = cluster.getAllBlockReports(poolId); for(int i = 0; i < blocks.size(); i++) { DataNode dn = datanodes.get(i); - Map map = blocks.get(i); - for(Map.Entry e : map.entrySet()) { + Map map = blocks.get(i); + for(Map.Entry e : map.entrySet()) { for(Block b : e.getValue()) { files.add(DataNodeTestUtils.getFile(dn, poolId, b.getBlockId())); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java index da2b8bbfe80..1f2fbbd8e4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestInjectionForSimulatedStorage.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.util.Time; import org.junit.Test; @@ -136,7 +137,7 @@ public class TestInjectionForSimulatedStorage { DFSTestUtil.createFile(cluster.getFileSystem(), testPath, filesize, filesize, blockSize, (short) numDataNodes, 0L); waitForBlockReplication(testFile, dfsClient.getNamenode(), numDataNodes, 20); - List> blocksList = cluster.getAllBlockReports(bpid); + List> blocksList = cluster.getAllBlockReports(bpid); cluster.shutdown(); cluster = null; @@ -157,7 +158,7 @@ public class TestInjectionForSimulatedStorage { .build(); cluster.waitActive(); Set uniqueBlocks = new HashSet(); - for(Map map : blocksList) { + for(Map map : blocksList) { for(BlockListAsLongs blockList : map.values()) { for(Block b : blockList) { uniqueBlocks.add(new Block(b)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index eebc5459655..f5b291e0ae6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -484,12 +485,9 @@ public class SimulatedFSDataset implements FsDatasetSpi { } @Override - public synchronized Map getBlockReports( + public synchronized Map getBlockReports( String bpid) { - Map reports = - new HashMap(); - reports.put(storage.storageUuid, getBlockReport(bpid)); - return reports; + return Collections.singletonMap(new DatanodeStorage(storage.storageUuid), getBlockReport(bpid)); } @Override // FsDatasetSpi diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java index 53b47e7d840..bd54edd0cee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java @@ -120,7 +120,7 @@ public class TestBlockReport { private static StorageBlockReport[] getBlockReports( DataNode dn, String bpid, boolean corruptOneBlockGs, boolean corruptOneBlockLen) { - Map perVolumeBlockLists = + Map perVolumeBlockLists = dn.getFSDataset().getBlockReports(bpid); // Send block report @@ -130,8 +130,8 @@ public class TestBlockReport { boolean corruptedLen = false; int reportIndex = 0; - for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { - String storageID = kvPair.getKey(); + for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { + DatanodeStorage dnStorage = kvPair.getKey(); BlockListAsLongs blockList = kvPair.getValue(); // Walk the list of blocks until we find one each to corrupt the @@ -150,8 +150,6 @@ public class TestBlockReport { } } - // Dummy DatanodeStorage object just for sending the block report. - DatanodeStorage dnStorage = new DatanodeStorage(storageID); reports[reportIndex++] = new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java index 30e23420ac9..646d33d13b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java @@ -154,7 +154,7 @@ public class TestDataNodeVolumeFailure { String bpid = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnR = dn.getDNRegistrationForBP(bpid); - Map perVolumeBlockLists = + Map perVolumeBlockLists = dn.getFSDataset().getBlockReports(bpid); // Send block report @@ -162,10 +162,9 @@ public class TestDataNodeVolumeFailure { new StorageBlockReport[perVolumeBlockLists.size()]; int reportIndex = 0; - for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { - String storageID = kvPair.getKey(); + for(Map.Entry kvPair : perVolumeBlockLists.entrySet()) { + DatanodeStorage dnStorage = kvPair.getKey(); BlockListAsLongs blockList = kvPair.getValue(); - DatanodeStorage dnStorage = new DatanodeStorage(storageID); reports[reportIndex++] = new StorageBlockReport(dnStorage, blockList.getBlockListAsLongs()); } From 87734ec939070877c80c13eb1836fad2797952aa Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Wed, 11 Dec 2013 01:09:13 +0000 Subject: [PATCH 52/76] Move HDFS-5257,HDFS-5427,HDFS-5443,HDFS-5476,HDFS-5425,HDFS-5474,HDFS-5504,HDFS-5428 into branch-2.3 section. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550011 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 48 ++++++++++----------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 23179b30672..9ebf2f34313 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -629,37 +629,13 @@ Release 2.4.0 - UNRELEASED HDFS-5400. DFS_CLIENT_MMAP_CACHE_THREAD_RUNS_PER_TIMEOUT constant is set to the wrong value. (Colin Patrick McCabe) - HDFS-5257. addBlock() retry should return LocatedBlock with locations else client - will get AIOBE. (Vinay via jing9) - - HDFS-5427. Not able to read deleted files from snapshot directly under - snapshottable dir after checkpoint and NN restart. (Vinay via jing9) - - HDFS-5443. Delete 0-sized block when deleting an under-construction file that - is included in snapshot. (jing9) - - HDFS-5476. Snapshot: clean the blocks/files/directories under a renamed - file/directory while deletion. (jing9) - - HDFS-5425. Renaming underconstruction file with snapshots can make NN failure on - restart. (jing9 and Vinay) - - HDFS-5474. Deletesnapshot can make Namenode in safemode on NN restarts. - (Sathish via jing9) - HDFS-5075. httpfs-config.sh calls out incorrect env script name (Timothy St. Clair via stevel) - HDFS-5504. In HA mode, OP_DELETE_SNAPSHOT is not decrementing the safemode threshold, - leads to NN safemode. (Vinay via jing9) - HDFS-5438. Flaws in block report processing can cause data loss. (kihwal) HDFS-5502. Fix HTTPS support in HsftpFileSystem. (Haohui Mai via jing9) - HDFS-5428. Under construction files deletion after snapshot+checkpoint+nn restart - leads nn safemode. (jing9) - HDFS-5552. Fix wrong information of "Cluster summay" in dfshealth.html. (Haohui Mai via jing9) @@ -804,6 +780,30 @@ Release 2.3.0 - UNRELEASED HDFS-5353. Short circuit reads fail when dfs.encrypt.data.transfer is enabled. (Colin Patrick McCabe via jing9) + HDFS-5257. addBlock() retry should return LocatedBlock with locations else client + will get AIOBE. (Vinay via jing9) + + HDFS-5427. Not able to read deleted files from snapshot directly under + snapshottable dir after checkpoint and NN restart. (Vinay via jing9) + + HDFS-5443. Delete 0-sized block when deleting an under-construction file that + is included in snapshot. (jing9) + + HDFS-5476. Snapshot: clean the blocks/files/directories under a renamed + file/directory while deletion. (jing9) + + HDFS-5425. Renaming underconstruction file with snapshots can make NN failure on + restart. (jing9 and Vinay) + + HDFS-5474. Deletesnapshot can make Namenode in safemode on NN restarts. + (Sathish via jing9) + + HDFS-5504. In HA mode, OP_DELETE_SNAPSHOT is not decrementing the safemode threshold, + leads to NN safemode. (Vinay via jing9) + + HDFS-5428. Under construction files deletion after snapshot+checkpoint+nn restart + leads nn safemode. (jing9) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES From 512a18a8d92305a34f3037064ceabdc5aff1f8bf Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 11 Dec 2013 01:32:22 +0000 Subject: [PATCH 53/76] HDFS-5074. Allow starting up from an fsimage checkpoint in the middle of a segment. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550016 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../bkjournal/BookKeeperJournalManager.java | 7 +- .../hdfs/qjournal/client/AsyncLogger.java | 2 +- .../hdfs/qjournal/client/AsyncLoggerSet.java | 4 +- .../qjournal/client/IPCLoggerChannel.java | 6 +- .../qjournal/client/QuorumJournalManager.java | 9 +- .../qjournal/protocol/QJournalProtocol.java | 4 +- ...JournalProtocolServerSideTranslatorPB.java | 1 - .../QJournalProtocolTranslatorPB.java | 3 +- .../hadoop/hdfs/qjournal/server/Journal.java | 7 +- .../qjournal/server/JournalNodeRpcServer.java | 4 +- .../server/namenode/BackupJournalManager.java | 2 +- .../hdfs/server/namenode/FSEditLog.java | 21 +-- .../server/namenode/FileJournalManager.java | 21 +-- .../hdfs/server/namenode/JournalSet.java | 15 +- .../hdfs/server/namenode/LogsPurgeable.java | 4 +- .../namenode/NNStorageRetentionManager.java | 2 +- .../server/namenode/SecondaryNameNode.java | 2 +- .../server/namenode/ha/BootstrapStandby.java | 2 +- .../server/namenode/ha/EditLogTailer.java | 2 +- .../src/main/proto/QJournalProtocol.proto | 2 +- .../apache/hadoop/hdfs/MiniDFSCluster.java | 3 +- .../hdfs/qjournal/MiniQJMHACluster.java | 134 ++++++++++++++++++ .../hadoop/hdfs/qjournal/TestNNWithQJM.java | 1 - .../client/TestQuorumJournalManager.java | 2 +- .../hdfs/server/namenode/NameNodeAdapter.java | 11 ++ .../namenode/TestFileJournalManager.java | 14 +- .../namenode/TestGenericJournalConf.java | 2 +- .../TestNNStorageRetentionManager.java | 7 +- .../ha/TestBootstrapStandbyWithQJM.java | 83 ++--------- .../namenode/ha/TestFailureToReadEdits.java | 67 ++++++--- 31 files changed, 263 insertions(+), 184 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9ebf2f34313..8517ae2a29b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -642,6 +642,9 @@ Release 2.4.0 - UNRELEASED HDFS-5533. Symlink delete/create should be treated as DELETE/CREATE in snapshot diff report. (Binglin Chang via jing9) + HDFS-5074. Allow starting up from an fsimage checkpoint in the middle of a + segment. (Todd Lipcon via atm) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java index ed57eeabeb7..5dc12811fe7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java @@ -500,14 +500,9 @@ public class BookKeeperJournalManager implements JournalManager { } } - public void selectInputStreams(Collection streams, - long fromTxId, boolean inProgressOk) throws IOException { - selectInputStreams(streams, fromTxId, inProgressOk, true); - } - @Override public void selectInputStreams(Collection streams, - long fromTxId, boolean inProgressOk, boolean forReading) + long fromTxId, boolean inProgressOk) throws IOException { List currentLedgerList = getLedgerList(fromTxId, inProgressOk); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java index f98cf0251ae..2f6baa3e4c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java @@ -109,7 +109,7 @@ interface AsyncLogger { * Fetch the list of edit logs available on the remote node. */ public ListenableFuture getEditLogManifest( - long fromTxnId, boolean forReading, boolean inProgressOk); + long fromTxnId, boolean inProgressOk); /** * Prepare recovery. See the HDFS-3077 design document for details. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java index 0abbd722189..d8918584d68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java @@ -261,13 +261,13 @@ class AsyncLoggerSet { } public QuorumCall getEditLogManifest( - long fromTxnId, boolean forReading, boolean inProgressOk) { + long fromTxnId, boolean inProgressOk) { Map> calls = Maps.newHashMap(); for (AsyncLogger logger : loggers) { ListenableFuture future = - logger.getEditLogManifest(fromTxnId, forReading, inProgressOk); + logger.getEditLogManifest(fromTxnId, inProgressOk); calls.put(logger, future); } return QuorumCall.create(calls); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java index b5ea5ce69e2..3731f5a7af7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java @@ -181,6 +181,7 @@ public class IPCLoggerChannel implements AsyncLogger { @Override public void close() { + QuorumJournalManager.LOG.info("Closing", new Exception()); // No more tasks may be submitted after this point. executor.shutdown(); if (proxy != null) { @@ -520,13 +521,12 @@ public class IPCLoggerChannel implements AsyncLogger { @Override public ListenableFuture getEditLogManifest( - final long fromTxnId, final boolean forReading, - final boolean inProgressOk) { + final long fromTxnId, final boolean inProgressOk) { return executor.submit(new Callable() { @Override public RemoteEditLogManifest call() throws IOException { GetEditLogManifestResponseProto ret = getProxy().getEditLogManifest( - journalId, fromTxnId, forReading, inProgressOk); + journalId, fromTxnId, inProgressOk); // Update the http port, since we need this to build URLs to any of the // returned logs. constructHttpServerURI(ret); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java index e0fcb53ba0f..9f2cd56b867 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java @@ -449,18 +449,13 @@ public class QuorumJournalManager implements JournalManager { public void close() throws IOException { loggers.close(); } - - public void selectInputStreams(Collection streams, - long fromTxnId, boolean inProgressOk) throws IOException { - selectInputStreams(streams, fromTxnId, inProgressOk, true); - } @Override public void selectInputStreams(Collection streams, - long fromTxnId, boolean inProgressOk, boolean forReading) throws IOException { + long fromTxnId, boolean inProgressOk) throws IOException { QuorumCall q = - loggers.getEditLogManifest(fromTxnId, forReading, inProgressOk); + loggers.getEditLogManifest(fromTxnId, inProgressOk); Map resps = loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs, "selectInputStreams"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java index 15ee76c6e9b..41600acdc92 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java @@ -123,14 +123,12 @@ public interface QJournalProtocol { /** * @param jid the journal from which to enumerate edits * @param sinceTxId the first transaction which the client cares about - * @param forReading whether or not the caller intends to read from the edit - * logs * @param inProgressOk whether or not to check the in-progress edit log * segment * @return a list of edit log segments since the given transaction ID. */ public GetEditLogManifestResponseProto getEditLogManifest(String jid, - long sinceTxId, boolean forReading, boolean inProgressOk) + long sinceTxId, boolean inProgressOk) throws IOException; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java index 50714040268..47d81005952 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java @@ -203,7 +203,6 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP return impl.getEditLogManifest( request.getJid().getIdentifier(), request.getSinceTxId(), - request.getForReading(), request.getInProgressOk()); } catch (IOException e) { throw new ServiceException(e); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java index 2df7d94bc5a..f111933842e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java @@ -228,14 +228,13 @@ public class QJournalProtocolTranslatorPB implements ProtocolMetaInterface, @Override public GetEditLogManifestResponseProto getEditLogManifest(String jid, - long sinceTxId, boolean forReading, boolean inProgressOk) + long sinceTxId, boolean inProgressOk) throws IOException { try { return rpcProxy.getEditLogManifest(NULL_CONTROLLER, GetEditLogManifestRequestProto.newBuilder() .setJid(convertJournalId(jid)) .setSinceTxId(sinceTxId) - .setForReading(forReading) .setInProgressOk(inProgressOk) .build()); } catch (ServiceException e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java index b68516b6c6a..cf83cef92c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java @@ -630,15 +630,12 @@ class Journal implements Closeable { * @see QJournalProtocol#getEditLogManifest(String, long) */ public RemoteEditLogManifest getEditLogManifest(long sinceTxId, - boolean forReading, boolean inProgressOk) throws IOException { + boolean inProgressOk) throws IOException { // No need to checkRequest() here - anyone may ask for the list // of segments. checkFormatted(); - // if this is for reading, ignore the in-progress editlog segment - inProgressOk = forReading ? false : inProgressOk; - List logs = fjm.getRemoteEditLogs(sinceTxId, forReading, - inProgressOk); + List logs = fjm.getRemoteEditLogs(sinceTxId, inProgressOk); if (inProgressOk) { RemoteEditLog log = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java index c30e16aaf97..5749cc17e47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java @@ -178,11 +178,11 @@ class JournalNodeRpcServer implements QJournalProtocol { @SuppressWarnings("deprecation") @Override public GetEditLogManifestResponseProto getEditLogManifest(String jid, - long sinceTxId, boolean forReading, boolean inProgressOk) + long sinceTxId, boolean inProgressOk) throws IOException { RemoteEditLogManifest manifest = jn.getOrCreateJournal(jid) - .getEditLogManifest(sinceTxId, forReading, inProgressOk); + .getEditLogManifest(sinceTxId, inProgressOk); return GetEditLogManifestResponseProto.newBuilder() .setManifest(PBHelper.convert(manifest)) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java index 46f43097747..5420b129cab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupJournalManager.java @@ -77,7 +77,7 @@ class BackupJournalManager implements JournalManager { @Override public void selectInputStreams(Collection streams, - long fromTxnId, boolean inProgressOk, boolean forReading) { + long fromTxnId, boolean inProgressOk) { // This JournalManager is never used for input. Therefore it cannot // return any transactions } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index e7019f4a52b..1abfcdfb5e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -286,7 +286,7 @@ public class FSEditLog implements LogsPurgeable { // Safety check: we should never start a segment if there are // newer txids readable. List streams = new ArrayList(); - journalSet.selectInputStreams(streams, segmentTxId, true, true); + journalSet.selectInputStreams(streams, segmentTxId, true); if (!streams.isEmpty()) { String error = String.format("Cannot start writing at txid %s " + "when there is a stream available for read: %s", @@ -1037,7 +1037,7 @@ public class FSEditLog implements LogsPurgeable { */ public synchronized RemoteEditLogManifest getEditLogManifest(long fromTxId) throws IOException { - return journalSet.getEditLogManifest(fromTxId, true); + return journalSet.getEditLogManifest(fromTxId); } /** @@ -1332,22 +1332,14 @@ public class FSEditLog implements LogsPurgeable { @Override public void selectInputStreams(Collection streams, - long fromTxId, boolean inProgressOk, boolean forReading) { - journalSet.selectInputStreams(streams, fromTxId, inProgressOk, forReading); + long fromTxId, boolean inProgressOk) throws IOException { + journalSet.selectInputStreams(streams, fromTxId, inProgressOk); } public Collection selectInputStreams( long fromTxId, long toAtLeastTxId) throws IOException { return selectInputStreams(fromTxId, toAtLeastTxId, null, true); } - - /** Select a list of input streams to load */ - public Collection selectInputStreams( - long fromTxId, long toAtLeastTxId, MetaRecoveryContext recovery, - boolean inProgressOk) throws IOException { - return selectInputStreams(fromTxId, toAtLeastTxId, recovery, inProgressOk, - true); - } /** * Select a list of input streams. @@ -1355,13 +1347,12 @@ public class FSEditLog implements LogsPurgeable { * @param fromTxId first transaction in the selected streams * @param toAtLeast the selected streams must contain this transaction * @param inProgessOk set to true if in-progress streams are OK - * @param forReading whether or not to use the streams to load the edit log */ public synchronized Collection selectInputStreams( long fromTxId, long toAtLeastTxId, MetaRecoveryContext recovery, - boolean inProgressOk, boolean forReading) throws IOException { + boolean inProgressOk) throws IOException { List streams = new ArrayList(); - selectInputStreams(streams, fromTxId, inProgressOk, forReading); + selectInputStreams(streams, fromTxId, inProgressOk); try { checkForGaps(streams, fromTxId, toAtLeastTxId, inProgressOk); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java index 77aca197ab6..09bddefc441 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java @@ -167,19 +167,13 @@ public class FileJournalManager implements JournalManager { /** * Find all editlog segments starting at or above the given txid. * @param fromTxId the txnid which to start looking - * @param forReading whether or not the caller intends to read from the edit - * logs * @param inProgressOk whether or not to include the in-progress edit log * segment * @return a list of remote edit logs * @throws IOException if edit logs cannot be listed. */ public List getRemoteEditLogs(long firstTxId, - boolean forReading, boolean inProgressOk) throws IOException { - // make sure not reading in-progress edit log, i.e., if forReading is true, - // we should ignore the in-progress edit log. - Preconditions.checkArgument(!(forReading && inProgressOk)); - + boolean inProgressOk) throws IOException { File currentDir = sd.getCurrentDir(); List allLogFiles = matchEditLogs(currentDir); List ret = Lists.newArrayListWithCapacity( @@ -192,14 +186,9 @@ public class FileJournalManager implements JournalManager { if (elf.getFirstTxId() >= firstTxId) { ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId)); } else if (elf.getFirstTxId() < firstTxId && firstTxId <= elf.getLastTxId()) { - // If the firstTxId is in the middle of an edit log segment - if (forReading) { - // Note that this behavior is different from getLogFiles below. - throw new IllegalStateException("Asked for firstTxId " + firstTxId - + " which is in the middle of file " + elf.file); - } else { - ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId)); - } + // If the firstTxId is in the middle of an edit log segment. Return this + // anyway and let the caller figure out whether it wants to use it. + ret.add(new RemoteEditLog(elf.firstTxId, elf.lastTxId)); } } @@ -260,7 +249,7 @@ public class FileJournalManager implements JournalManager { @Override synchronized public void selectInputStreams( Collection streams, long fromTxId, - boolean inProgressOk, boolean forReading) throws IOException { + boolean inProgressOk) throws IOException { List elfs = matchEditLogs(sd.getCurrentDir()); LOG.debug(this + ": selecting input streams starting at " + fromTxId + (inProgressOk ? " (inProgress ok) " : " (excluding inProgress) ") + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java index 1d43cb73527..b117606f8cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java @@ -233,12 +233,10 @@ public class JournalSet implements JournalManager { * may not be sorted-- this is up to the caller. * @param fromTxId The transaction ID to start looking for streams at * @param inProgressOk Should we consider unfinalized streams? - * @param forReading Whether or not the caller intends to read from - * the returned streams. */ @Override public void selectInputStreams(Collection streams, - long fromTxId, boolean inProgressOk, boolean forReading) { + long fromTxId, boolean inProgressOk) { final PriorityQueue allStreams = new PriorityQueue(64, EDIT_LOG_INPUT_STREAM_COMPARATOR); @@ -248,8 +246,7 @@ public class JournalSet implements JournalManager { continue; } try { - jas.getManager().selectInputStreams(allStreams, fromTxId, inProgressOk, - forReading); + jas.getManager().selectInputStreams(allStreams, fromTxId, inProgressOk); } catch (IOException ioe) { LOG.warn("Unable to determine input streams from " + jas.getManager() + ". Skipping.", ioe); @@ -582,20 +579,20 @@ public class JournalSet implements JournalManager { /** * Return a manifest of what finalized edit logs are available. All available - * edit logs are returned starting from the transaction id passed. + * edit logs are returned starting from the transaction id passed. If + * 'fromTxId' falls in the middle of a log, that log is returned as well. * * @param fromTxId Starting transaction id to read the logs. * @return RemoteEditLogManifest object. */ - public synchronized RemoteEditLogManifest getEditLogManifest(long fromTxId, - boolean forReading) { + public synchronized RemoteEditLogManifest getEditLogManifest(long fromTxId) { // Collect RemoteEditLogs available from each FileJournalManager List allLogs = Lists.newArrayList(); for (JournalAndStream j : journals) { if (j.getManager() instanceof FileJournalManager) { FileJournalManager fjm = (FileJournalManager)j.getManager(); try { - allLogs.addAll(fjm.getRemoteEditLogs(fromTxId, forReading, false)); + allLogs.addAll(fjm.getRemoteEditLogs(fromTxId, false)); } catch (Throwable t) { LOG.warn("Cannot list edit logs in " + fjm, t); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java index 4261825d723..f997fa0b35e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java @@ -42,13 +42,11 @@ interface LogsPurgeable { * * @param fromTxId the first transaction id we want to read * @param inProgressOk whether or not in-progress streams should be returned - * @param forReading whether or not the caller intends to read from the edit logs - * * @return a list of streams * @throws IOException if the underlying storage has an error or is otherwise * inaccessible */ void selectInputStreams(Collection streams, - long fromTxId, boolean inProgressOk, boolean forReading) throws IOException; + long fromTxId, boolean inProgressOk) throws IOException; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java index 6b545a65436..75d1bd82fec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java @@ -108,7 +108,7 @@ public class NNStorageRetentionManager { long purgeLogsFrom = Math.max(0, minimumRequiredTxId - numExtraEditsToRetain); ArrayList editLogs = new ArrayList(); - purgeableLogs.selectInputStreams(editLogs, purgeLogsFrom, false, false); + purgeableLogs.selectInputStreams(editLogs, purgeLogsFrom, false); Collections.sort(editLogs, new Comparator() { @Override public int compare(EditLogInputStream a, EditLogInputStream b) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index 6d136e87558..c3d9b93efe1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -827,7 +827,7 @@ public class SecondaryNameNode implements Runnable { @Override public void selectInputStreams(Collection streams, - long fromTxId, boolean inProgressOk, boolean forReading) { + long fromTxId, boolean inProgressOk) { Iterator iter = storage.dirIterator(); while (iter.hasNext()) { StorageDirectory dir = iter.next(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java index 490b1c1f9fd..484eb45cc0f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java @@ -228,7 +228,7 @@ public class BootstrapStandby implements Tool, Configurable { try { Collection streams = image.getEditLog().selectInputStreams( - firstTxIdInLogs, curTxIdOnOtherNode, null, true, false); + firstTxIdInLogs, curTxIdOnOtherNode, null, true); for (EditLogInputStream stream : streams) { IOUtils.closeStream(stream); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java index 8e0739afb1e..b9ae43d576d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java @@ -165,7 +165,7 @@ public class EditLogTailer { } @VisibleForTesting - void setEditLog(FSEditLog editLog) { + public void setEditLog(FSEditLog editLog) { this.editLog = editLog; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto index 8ee8ca5558b..ae963751221 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto @@ -178,7 +178,7 @@ message GetEditLogManifestRequestProto { required JournalIdProto jid = 1; required uint64 sinceTxId = 2; // Transaction ID // Whether or not the client will be reading from the returned streams. - optional bool forReading = 3 [default = true]; + // optional bool forReading = 3 [default = true]; optional bool inProgressOk = 4 [default = false]; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 72f053c909e..84d996bad17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -1487,8 +1487,9 @@ public class MiniDFSCluster { */ public synchronized void restartNameNodes() throws IOException { for (int i = 0; i < nameNodes.length; i++) { - restartNameNode(i); + restartNameNode(i, false); } + waitActive(); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java new file mode 100644 index 00000000000..e5b636c562f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.qjournal; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; + +import java.io.IOException; +import java.net.URI; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; + +public class MiniQJMHACluster { + private MiniDFSCluster cluster; + private MiniJournalCluster journalCluster; + private final Configuration conf; + + private static String NAMESERVICE = "ns1"; + private static final String NN1 = "nn1"; + private static final String NN2 = "nn2"; + private static final int NN1_IPC_PORT = 10000; + private static final int NN1_INFO_PORT = 10001; + private static final int NN2_IPC_PORT = 10002; + private static final int NN2_INFO_PORT = 10003; + + public static class Builder { + private final Configuration conf; + private final MiniDFSCluster.Builder dfsBuilder; + + public Builder(Configuration conf) { + this.conf = conf; + this.dfsBuilder = new MiniDFSCluster.Builder(conf); + } + + public MiniDFSCluster.Builder getDfsBuilder() { + return dfsBuilder; + } + + public MiniQJMHACluster build() throws IOException { + return new MiniQJMHACluster(this); + } + } + + public static MiniDFSNNTopology createDefaultTopology() { + return new MiniDFSNNTopology() + .addNameservice(new MiniDFSNNTopology.NSConf(NAMESERVICE).addNN( + new MiniDFSNNTopology.NNConf("nn1").setIpcPort(NN1_IPC_PORT) + .setHttpPort(NN1_INFO_PORT)).addNN( + new MiniDFSNNTopology.NNConf("nn2").setIpcPort(NN2_IPC_PORT) + .setHttpPort(NN2_INFO_PORT))); + } + + private MiniQJMHACluster(Builder builder) throws IOException { + this.conf = builder.conf; + // start 3 journal nodes + journalCluster = new MiniJournalCluster.Builder(conf).format(true) + .build(); + URI journalURI = journalCluster.getQuorumJournalURI(NAMESERVICE); + + // start cluster with 2 NameNodes + MiniDFSNNTopology topology = createDefaultTopology(); + + initHAConf(journalURI, builder.conf); + + // First start up the NNs just to format the namespace. The MinIDFSCluster + // has no way to just format the NameNodes without also starting them. + cluster = builder.dfsBuilder.nnTopology(topology) + .manageNameDfsSharedDirs(false).build(); + cluster.waitActive(); + cluster.shutdown(); + + // initialize the journal nodes + Configuration confNN0 = cluster.getConfiguration(0); + NameNode.initializeSharedEdits(confNN0, true); + + // restart the cluster + cluster.restartNameNodes(); + } + + private Configuration initHAConf(URI journalURI, Configuration conf) { + conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, + journalURI.toString()); + + String address1 = "127.0.0.1:" + NN1_IPC_PORT; + String address2 = "127.0.0.1:" + NN2_IPC_PORT; + conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, + NAMESERVICE, NN1), address1); + conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, + NAMESERVICE, NN2), address2); + conf.set(DFSConfigKeys.DFS_NAMESERVICES, NAMESERVICE); + conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, NAMESERVICE), + NN1 + "," + NN2); + conf.set(DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + NAMESERVICE, + ConfiguredFailoverProxyProvider.class.getName()); + conf.set("fs.defaultFS", "hdfs://" + NAMESERVICE); + + return conf; + } + + public MiniDFSCluster getDfsCluster() { + return cluster; + } + + public MiniJournalCluster getJournalCluster() { + return journalCluster; + } + + public void shutdown() throws IOException { + cluster.shutdown(); + journalCluster.shutdown(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java index a7eed468e62..132ba748bad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestNNWithQJM.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.qjournal; import static org.junit.Assert.*; -import static org.junit.Assume.*; import java.io.File; import java.io.IOException; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java index 990d3ec2bff..d513788a583 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java @@ -916,7 +916,7 @@ public class TestQuorumJournalManager { NNStorage.getFinalizedEditsFileName(41, 50)); ArrayList streams = new ArrayList(); - qjm.selectInputStreams(streams, 25, false, false); + qjm.selectInputStreams(streams, 25, false); verifyEdits(streams, 25, 50); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java index b83adecd194..4f5c2b6732b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.MkdirOp; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.SafeModeInfo; import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease; +import org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse; import org.apache.hadoop.ipc.Server; @@ -180,6 +181,16 @@ public class NameNodeAdapter { return spy; } + public static FSEditLog spyOnEditLog(NameNode nn) { + FSEditLog spyEditLog = spy(nn.getNamesystem().getFSImage().getEditLog()); + nn.getFSImage().setEditLogForTesting(spyEditLog); + EditLogTailer tailer = nn.getNamesystem().getEditLogTailer(); + if (tailer != null) { + tailer.setEditLog(spyEditLog); + } + return spyEditLog; + } + public static JournalSet spyOnJournalSet(NameNode nn) { FSEditLog editLog = nn.getFSImage().getEditLog(); JournalSet js = Mockito.spy(editLog.getJournalSet()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java index 342a167cac3..cb7c3e4388b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java @@ -82,7 +82,7 @@ public class TestFileJournalManager { final PriorityQueue allStreams = new PriorityQueue(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); - jm.selectInputStreams(allStreams, fromTxId, inProgressOk, true); + jm.selectInputStreams(allStreams, fromTxId, inProgressOk); EditLogInputStream elis = null; try { while ((elis = allStreams.poll()) != null) { @@ -378,14 +378,8 @@ public class TestFileJournalManager { FileJournalManager fjm = new FileJournalManager(conf, sd, null); assertEquals("[1,100],[101,200],[1001,1100]", getLogsAsString(fjm, 1)); assertEquals("[101,200],[1001,1100]", getLogsAsString(fjm, 101)); + assertEquals("[101,200],[1001,1100]", getLogsAsString(fjm, 150)); assertEquals("[1001,1100]", getLogsAsString(fjm, 201)); - try { - assertEquals("[]", getLogsAsString(fjm, 150)); - fail("Did not throw when asking for a txn in the middle of a log"); - } catch (IllegalStateException ioe) { - GenericTestUtils.assertExceptionContains( - "150 which is in the middle", ioe); - } assertEquals("Asking for a newer log than exists should return empty list", "", getLogsAsString(fjm, 9999)); } @@ -404,7 +398,7 @@ public class TestFileJournalManager { final PriorityQueue allStreams = new PriorityQueue(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR); - jm.selectInputStreams(allStreams, txId, inProgressOk, true); + jm.selectInputStreams(allStreams, txId, inProgressOk); EditLogInputStream elis = null, ret; try { while ((elis = allStreams.poll()) != null) { @@ -482,6 +476,6 @@ public class TestFileJournalManager { private static String getLogsAsString( FileJournalManager fjm, long firstTxId) throws IOException { - return Joiner.on(",").join(fjm.getRemoteEditLogs(firstTxId, true, false)); + return Joiner.on(",").join(fjm.getRemoteEditLogs(firstTxId, false)); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java index a4f594bc895..43bd7a4153a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGenericJournalConf.java @@ -170,7 +170,7 @@ public class TestGenericJournalConf { @Override public void selectInputStreams(Collection streams, - long fromTxnId, boolean inProgressOk, boolean forReading) { + long fromTxnId, boolean inProgressOk) { } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java index 2d42cd0b228..24a9cd04f4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java @@ -324,7 +324,7 @@ public class TestNNStorageRetentionManager { } @SuppressWarnings("unchecked") - public FSEditLog mockEditLog(StoragePurger purger) { + public FSEditLog mockEditLog(StoragePurger purger) throws IOException { final List jms = Lists.newArrayList(); final JournalSet journalSet = new JournalSet(0); for (FakeRoot root : dirRoots.values()) { @@ -360,12 +360,11 @@ public class TestNNStorageRetentionManager { public Void answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); journalSet.selectInputStreams((Collection)args[0], - (long)((Long)args[1]), (boolean)((Boolean)args[2]), - (boolean)((Boolean)args[3])); + (long)((Long)args[1]), (boolean)((Boolean)args[2])); return null; } }).when(mockLog).selectInputStreams(Mockito.anyCollection(), - Mockito.anyLong(), Mockito.anyBoolean(), Mockito.anyBoolean()); + Mockito.anyLong(), Mockito.anyBoolean()); return mockLog; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java index e618c9a5ed0..c7756ca5bb8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java @@ -17,24 +17,18 @@ */ package org.apache.hadoop.hdfs.server.namenode.ha; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.junit.Assert.assertEquals; import java.io.IOException; -import java.net.URI; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; +import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -44,51 +38,23 @@ import com.google.common.collect.ImmutableList; /** * Test BootstrapStandby when QJM is used for shared edits. */ -public class TestBootstrapStandbyWithQJM { - - private static final String NAMESERVICE = "ns1"; - private static final String NN1 = "nn1"; - private static final String NN2 = "nn2"; - private static final int NUM_JN = 3; - private static final int NN1_IPC_PORT = 10000; - private static final int NN1_INFO_PORT = 10001; - private static final int NN2_IPC_PORT = 10002; - private static final int NN2_INFO_PORT = 10003; - +public class TestBootstrapStandbyWithQJM { + private MiniQJMHACluster miniQjmHaCluster; private MiniDFSCluster cluster; private MiniJournalCluster jCluster; @Before public void setup() throws Exception { - // start 3 journal nodes - jCluster = new MiniJournalCluster.Builder(new Configuration()).format(true) - .numJournalNodes(NUM_JN).build(); - URI journalURI = jCluster.getQuorumJournalURI(NAMESERVICE); - - // start cluster with 2 NameNodes - MiniDFSNNTopology topology = new MiniDFSNNTopology() - .addNameservice(new MiniDFSNNTopology.NSConf(NAMESERVICE).addNN( - new MiniDFSNNTopology.NNConf("nn1").setIpcPort(NN1_IPC_PORT) - .setHttpPort(NN1_INFO_PORT)).addNN( - new MiniDFSNNTopology.NNConf("nn2").setIpcPort(NN2_IPC_PORT) - .setHttpPort(NN2_INFO_PORT))); - - Configuration conf = initHAConf(journalURI); - cluster = new MiniDFSCluster.Builder(conf).nnTopology(topology) - .numDataNodes(1).manageNameDfsSharedDirs(false).build(); - cluster.waitActive(); - - Configuration confNN0 = new Configuration(conf); - cluster.shutdown(); - // initialize the journal nodes - confNN0.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1"); - NameNode.initializeSharedEdits(confNN0, true); - - // restart the cluster - cluster = new MiniDFSCluster.Builder(conf).format(false) - .nnTopology(topology).numDataNodes(1).manageNameDfsSharedDirs(false) - .build(); - cluster.waitActive(); + Configuration conf = new Configuration(); + // Turn off IPC client caching, so that the suite can handle + // the restart of the daemons between test cases. + conf.setInt( + CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, + 0); + + miniQjmHaCluster = new MiniQJMHACluster.Builder(conf).build(); + cluster = miniQjmHaCluster.getDfsCluster(); + jCluster = miniQjmHaCluster.getJournalCluster(); // make nn0 active cluster.transitionToActive(0); @@ -109,27 +75,6 @@ public class TestBootstrapStandbyWithQJM { } } - private Configuration initHAConf(URI journalURI) { - Configuration conf = new Configuration(); - conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, - journalURI.toString()); - - String address1 = "127.0.0.1:" + NN1_IPC_PORT; - String address2 = "127.0.0.1:" + NN2_IPC_PORT; - conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, - NAMESERVICE, NN1), address1); - conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, - NAMESERVICE, NN2), address2); - conf.set(DFSConfigKeys.DFS_NAMESERVICES, NAMESERVICE); - conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, NAMESERVICE), - NN1 + "," + NN2); - conf.set(DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + NAMESERVICE, - ConfiguredFailoverProxyProvider.class.getName()); - conf.set("fs.defaultFS", "hdfs://" + NAMESERVICE); - - return conf; - } - /** BootstrapStandby when the existing NN is standby */ @Test public void testBootstrapStandbyWithStandbyNN() throws Exception { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java index 5ec7f7e1c1e..8bcc6d72a51 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; @@ -37,6 +38,8 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster; +import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster.Builder; import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; import org.apache.hadoop.hdfs.server.namenode.FSEditLog; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; @@ -48,23 +51,49 @@ import org.apache.hadoop.util.ExitUtil.ExitException; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import com.google.common.collect.ImmutableList; +@RunWith(Parameterized.class) public class TestFailureToReadEdits { private static final String TEST_DIR1 = "/test1"; private static final String TEST_DIR2 = "/test2"; private static final String TEST_DIR3 = "/test3"; + private final TestType clusterType; private Configuration conf; private MiniDFSCluster cluster; + private MiniQJMHACluster miniQjmHaCluster; // for QJM case only private NameNode nn0; private NameNode nn1; private FileSystem fs; + private static enum TestType { + SHARED_DIR_HA, + QJM_HA; + }; + + /** + * Run this suite of tests both for QJM-based HA and for file-based + * HA. + */ + @Parameters + public static Iterable data() { + return Arrays.asList(new Object[][] { + { TestType.SHARED_DIR_HA }, + { TestType.QJM_HA } }); + } + + public TestFailureToReadEdits(TestType clusterType) { + this.clusterType = clusterType; + } + @Before public void setUpCluster() throws Exception { conf = new Configuration(); @@ -74,16 +103,19 @@ public class TestFailureToReadEdits { conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); HAUtil.setAllowStandbyReads(conf, true); - MiniDFSNNTopology topology = new MiniDFSNNTopology() - .addNameservice(new MiniDFSNNTopology.NSConf("ns1") - .addNN(new MiniDFSNNTopology.NNConf("nn1").setHttpPort(10041)) - .addNN(new MiniDFSNNTopology.NNConf("nn2").setHttpPort(10042))); - cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(topology) - .numDataNodes(0) - .checkExitOnShutdown(false) - .build(); - + if (clusterType == TestType.SHARED_DIR_HA) { + MiniDFSNNTopology topology = MiniQJMHACluster.createDefaultTopology(); + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(topology) + .numDataNodes(0) + .checkExitOnShutdown(false) + .build(); + } else { + Builder builder = new MiniQJMHACluster.Builder(conf); + builder.getDfsBuilder().numDataNodes(0).checkExitOnShutdown(false); + miniQjmHaCluster = builder.build(); + cluster = miniQjmHaCluster.getDfsCluster(); + } cluster.waitActive(); nn0 = cluster.getNameNode(0); @@ -99,8 +131,14 @@ public class TestFailureToReadEdits { fs.close(); } - if (cluster != null) { - cluster.shutdown(); + if (clusterType == TestType.SHARED_DIR_HA) { + if (cluster != null) { + cluster.shutdown(); + } + } else { + if (miniQjmHaCluster != null) { + miniQjmHaCluster.shutdown(); + } } } @@ -259,13 +297,10 @@ public class TestFailureToReadEdits { } private LimitedEditLogAnswer causeFailureOnEditLogRead() throws IOException { - FSEditLog spyEditLog = spy(nn1.getNamesystem().getEditLogTailer() - .getEditLog()); + FSEditLog spyEditLog = NameNodeAdapter.spyOnEditLog(nn1); LimitedEditLogAnswer answer = new LimitedEditLogAnswer(); doAnswer(answer).when(spyEditLog).selectInputStreams( anyLong(), anyLong(), (MetaRecoveryContext)anyObject(), anyBoolean()); - nn1.getNamesystem().getEditLogTailer().setEditLog(spyEditLog); - return answer; } From e408ef35e97cad66e909bb760096b19d1bf7cfcb Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 11 Dec 2013 02:01:54 +0000 Subject: [PATCH 54/76] Move entry for HDFS-5074 to correct section. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550027 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8517ae2a29b..9f90db1ac95 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -642,9 +642,6 @@ Release 2.4.0 - UNRELEASED HDFS-5533. Symlink delete/create should be treated as DELETE/CREATE in snapshot diff report. (Binglin Chang via jing9) - HDFS-5074. Allow starting up from an fsimage checkpoint in the middle of a - segment. (Todd Lipcon via atm) - Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -807,6 +804,9 @@ Release 2.3.0 - UNRELEASED HDFS-5428. Under construction files deletion after snapshot+checkpoint+nn restart leads nn safemode. (jing9) + HDFS-5074. Allow starting up from an fsimage checkpoint in the middle of a + segment. (Todd Lipcon via atm) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES From 732975be82357938445fa6078bcd46b868e89a37 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Wed, 11 Dec 2013 02:31:57 +0000 Subject: [PATCH 55/76] Move HDFS-5283 to section branch-2.3.0 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550032 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9f90db1ac95..de970ba7d6a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -616,9 +616,6 @@ Release 2.4.0 - UNRELEASED HDFS-5352. Server#initLog() doesn't close InputStream in httpfs. (Ted Yu via jing9) - HDFS-5283. Under construction blocks only inside snapshots should not be - counted in safemode threshhold. (Vinay via szetszwo) - HDFS-4376. Fix race conditions in Balancer. (Junping Du via szetszwo) HDFS-5375. hdfs.cmd does not expose several snapshot commands. (cnauroth) @@ -780,6 +777,9 @@ Release 2.3.0 - UNRELEASED HDFS-5353. Short circuit reads fail when dfs.encrypt.data.transfer is enabled. (Colin Patrick McCabe via jing9) + HDFS-5283. Under construction blocks only inside snapshots should not be + counted in safemode threshhold. (Vinay via szetszwo) + HDFS-5257. addBlock() retry should return LocatedBlock with locations else client will get AIOBE. (Vinay via jing9) From 327a995561270bc01c024e38ddd1063c42ff2e3c Mon Sep 17 00:00:00 2001 From: Junping Du Date: Wed, 11 Dec 2013 08:51:12 +0000 Subject: [PATCH 56/76] HDFS-5580. Fix infinite loop in Balancer.waitForMoveCompletion. (Binglin Chang via junping_du) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550074 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/server/balancer/Balancer.java | 31 ++++++++++--------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index de970ba7d6a..6090ef78e4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -639,6 +639,9 @@ Release 2.4.0 - UNRELEASED HDFS-5533. Symlink delete/create should be treated as DELETE/CREATE in snapshot diff report. (Binglin Chang via jing9) + HDFS-5580. Fix infinite loop in Balancer.waitForMoveCompletion. + (Binglin Chang via junping_du) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index befdd90a79e..ff09ca54dd1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -292,26 +292,27 @@ public class Balancer { */ private boolean chooseProxySource() { final DatanodeInfo targetDN = target.getDatanode(); - boolean find = false; - for (BalancerDatanode loc : block.getLocations()) { - // check if there is replica which is on the same rack with the target - if (cluster.isOnSameRack(loc.getDatanode(), targetDN) && addTo(loc)) { - find = true; - // if cluster is not nodegroup aware or the proxy is on the same - // nodegroup with target, then we already find the nearest proxy - if (!cluster.isNodeGroupAware() - || cluster.isOnSameNodeGroup(loc.getDatanode(), targetDN)) { + // if node group is supported, first try add nodes in the same node group + if (cluster.isNodeGroupAware()) { + for (BalancerDatanode loc : block.getLocations()) { + if (cluster.isOnSameNodeGroup(loc.getDatanode(), targetDN) && addTo(loc)) { return true; } } - - if (!find) { - // find out a non-busy replica out of rack of target - find = addTo(loc); + } + // check if there is replica which is on the same rack with the target + for (BalancerDatanode loc : block.getLocations()) { + if (cluster.isOnSameRack(loc.getDatanode(), targetDN) && addTo(loc)) { + return true; } } - - return find; + // find out a non-busy replica + for (BalancerDatanode loc : block.getLocations()) { + if (addTo(loc)) { + return true; + } + } + return false; } // add a BalancerDatanode as proxy source for specific block movement From 94e2e78ab751d1da775d0acf6e5ee6c0694ed746 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Wed, 11 Dec 2013 15:13:46 +0000 Subject: [PATCH 57/76] YARN-1481. Move internal services logic from AdminService to ResourceManager. (vinodkv via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550167 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../server/resourcemanager/AdminService.java | 77 +++------------- .../server/resourcemanager/RMContext.java | 7 +- .../server/resourcemanager/RMContextImpl.java | 30 ++++++- .../resourcemanager/ResourceManager.java | 89 ++++++++++++++----- 5 files changed, 120 insertions(+), 86 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c0d60d51cb2..7edd4bba467 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -154,6 +154,9 @@ Release 2.4.0 - UNRELEASED YARN-1378. Implemented a cleaner of old finished applications from the RM state-store. (Jian He via vinodkv) + YARN-1481. Move internal services logic from AdminService to ResourceManager. + (vinodkv via kasha) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index e78f002f880..18ffee595e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; import java.net.InetSocketAddress; -import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.BlockingService; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -43,7 +41,6 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.RMNotYetActiveException; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -66,6 +63,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMapp import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse; import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; +import com.google.protobuf.BlockingService; + public class AdminService extends AbstractService implements HAServiceProtocol, ResourceManagerAdministrationProtocol { @@ -73,10 +72,6 @@ public class AdminService extends AbstractService implements private final RMContext rmContext; private final ResourceManager rm; - @VisibleForTesting - protected HAServiceProtocol.HAServiceState - haState = HAServiceProtocol.HAServiceState.INITIALIZING; - boolean haEnabled; private Server server; private InetSocketAddress masterServiceAddress; @@ -93,13 +88,6 @@ public class AdminService extends AbstractService implements @Override public synchronized void serviceInit(Configuration conf) throws Exception { - haEnabled = HAUtil.isHAEnabled(conf); - if (haEnabled) { - HAUtil.verifyAndSetConfiguration(conf); - rm.setConf(conf); - } - rm.createAndInitActiveServices(); - masterServiceAddress = conf.getSocketAddr( YarnConfiguration.RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, @@ -112,11 +100,6 @@ public class AdminService extends AbstractService implements @Override protected synchronized void serviceStart() throws Exception { - if (haEnabled) { - transitionToStandby(true); - } else { - transitionToActive(); - } startServer(); super.serviceStart(); } @@ -124,8 +107,6 @@ public class AdminService extends AbstractService implements @Override protected synchronized void serviceStop() throws Exception { stopServer(); - transitionToStandby(false); - haState = HAServiceState.STOPPING; super.serviceStop(); } @@ -145,7 +126,7 @@ public class AdminService extends AbstractService implements refreshServiceAcls(conf, new RMPolicyProvider()); } - if (haEnabled) { + if (rmContext.isHAEnabled()) { RPC.setProtocolEngine(conf, HAServiceProtocolPB.class, ProtobufRpcEngine.class); @@ -182,39 +163,27 @@ public class AdminService extends AbstractService implements } private synchronized boolean isRMActive() { - return HAServiceState.ACTIVE == haState; + return HAServiceState.ACTIVE == rmContext.getHAServiceState(); } @Override public synchronized void monitorHealth() throws IOException { checkAccess("monitorHealth"); - if (haState == HAServiceProtocol.HAServiceState.ACTIVE && !rm.areActiveServicesRunning()) { + if (isRMActive() && !rm.areActiveServicesRunning()) { throw new HealthCheckFailedException( "Active ResourceManager services are not running!"); } } - synchronized void transitionToActive() throws Exception { - if (haState == HAServiceProtocol.HAServiceState.ACTIVE) { - LOG.info("Already in active state"); - return; - } - - LOG.info("Transitioning to active"); - rm.startActiveServices(); - haState = HAServiceProtocol.HAServiceState.ACTIVE; - LOG.info("Transitioned to active"); - } - @Override - public synchronized void transitionToActive(HAServiceProtocol.StateChangeRequestInfo reqInfo) - throws IOException { + public synchronized void transitionToActive( + HAServiceProtocol.StateChangeRequestInfo reqInfo) throws IOException { UserGroupInformation user = checkAccess("transitionToActive"); // TODO (YARN-1177): When automatic failover is enabled, // check if transition should be allowed for this request try { - transitionToActive(); + rm.transitionToActive(); RMAuditLogger.logSuccess(user.getShortUserName(), "transitionToActive", "RMHAProtocolService"); } catch (Exception e) { @@ -226,32 +195,14 @@ public class AdminService extends AbstractService implements } } - synchronized void transitionToStandby(boolean initialize) - throws Exception { - if (haState == HAServiceProtocol.HAServiceState.STANDBY) { - LOG.info("Already in standby state"); - return; - } - - LOG.info("Transitioning to standby"); - if (haState == HAServiceProtocol.HAServiceState.ACTIVE) { - rm.stopActiveServices(); - if (initialize) { - rm.createAndInitActiveServices(); - } - } - haState = HAServiceProtocol.HAServiceState.STANDBY; - LOG.info("Transitioned to standby"); - } - @Override - public synchronized void transitionToStandby(HAServiceProtocol.StateChangeRequestInfo reqInfo) - throws IOException { + public synchronized void transitionToStandby( + HAServiceProtocol.StateChangeRequestInfo reqInfo) throws IOException { UserGroupInformation user = checkAccess("transitionToStandby"); // TODO (YARN-1177): When automatic failover is enabled, // check if transition should be allowed for this request try { - transitionToStandby(true); + rm.transitionToStandby(true); RMAuditLogger.logSuccess(user.getShortUserName(), "transitionToStandby", "RMHAProtocolService"); } catch (Exception e) { @@ -266,15 +217,15 @@ public class AdminService extends AbstractService implements @Override public synchronized HAServiceStatus getServiceStatus() throws IOException { checkAccess("getServiceState"); + HAServiceState haState = rmContext.getHAServiceState(); HAServiceStatus ret = new HAServiceStatus(haState); - if (haState == HAServiceProtocol.HAServiceState.ACTIVE || haState == - HAServiceProtocol.HAServiceState.STANDBY) { + if (isRMActive() || haState == HAServiceProtocol.HAServiceState.STANDBY) { ret.setReadyToBecomeActive(); } else { ret.setNotReadyToBecomeActive("State is " + haState); } return ret; - } + } @Override public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index a93fcb13d76..1ddb1b48a52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.util.concurrent.ConcurrentMap; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.event.Dispatcher; @@ -42,7 +43,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenS public interface RMContext { Dispatcher getDispatcher(); - + + boolean isHAEnabled(); + + HAServiceState getHAServiceState(); + RMStateStore getStateStore(); ConcurrentMap getRMApps(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index f98b49efc8d..ec90b4a27d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -35,8 +37,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; -import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; import com.google.common.annotations.VisibleForTesting; @@ -54,6 +56,10 @@ public class RMContextImpl implements RMContext { private final ConcurrentMap inactiveNodes = new ConcurrentHashMap(); + private boolean isHAEnabled; + private HAServiceState haServiceState = + HAServiceProtocol.HAServiceState.INITIALIZING; + private AMLivelinessMonitor amLivelinessMonitor; private AMLivelinessMonitor amFinishingMonitor; private RMStateStore stateStore = null; @@ -211,6 +217,16 @@ public class RMContextImpl implements RMContext { return resourceTrackerService; } + void setHAEnabled(boolean isHAEnabled) { + this.isHAEnabled = isHAEnabled; + } + + void setHAServiceState(HAServiceState haServiceState) { + synchronized (haServiceState) { + this.haServiceState = haServiceState; + } + } + void setDispatcher(Dispatcher dispatcher) { this.rmDispatcher = dispatcher; } @@ -290,4 +306,16 @@ public class RMContextImpl implements RMContext { ResourceTrackerService resourceTrackerService) { this.resourceTrackerService = resourceTrackerService; } + + @Override + public boolean isHAEnabled() { + return isHAEnabled; + } + + @Override + public HAServiceState getHAServiceState() { + synchronized (haServiceState) { + return haServiceState; + } + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 597d18c112e..e44207796ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -27,6 +27,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; @@ -43,6 +45,7 @@ import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; @@ -188,6 +191,12 @@ public class ResourceManager extends CompositeService implements Recoverable { addService(adminService); rmContext.setRMAdminService(adminService); + this.rmContext.setHAEnabled(HAUtil.isHAEnabled(conf)); + if (this.rmContext.isHAEnabled()) { + HAUtil.verifyAndSetConfiguration(conf); + } + createAndInitActiveServices(); + super.serviceInit(conf); } @@ -217,9 +226,8 @@ public class ResourceManager extends CompositeService implements Recoverable { } protected RMStateStoreOperationFailedEventDispatcher - createRMStateStoreOperationFailedEventDispatcher() { - return new RMStateStoreOperationFailedEventDispatcher( - rmContext.getRMAdminService()); + createRMStateStoreOperationFailedEventDispatcher() { + return new RMStateStoreOperationFailedEventDispatcher(rmContext, this); } protected Dispatcher createDispatcher() { @@ -655,11 +663,14 @@ public class ResourceManager extends CompositeService implements Recoverable { @Private public static class RMStateStoreOperationFailedEventDispatcher implements EventHandler { - private final AdminService adminService; - public RMStateStoreOperationFailedEventDispatcher( - AdminService adminService) { - this.adminService = adminService; + private final RMContext rmContext; + private final ResourceManager rm; + + public RMStateStoreOperationFailedEventDispatcher(RMContext rmContext, + ResourceManager resourceManager) { + this.rmContext = rmContext; + this.rm = resourceManager; } @Override @@ -671,16 +682,14 @@ public class ResourceManager extends CompositeService implements Recoverable { } if (event.getType() == RMStateStoreOperationFailedEventType.FENCED) { LOG.info("RMStateStore has been fenced"); - synchronized(adminService) { - if (adminService.haEnabled) { - try { - // Transition to standby and reinit active services - LOG.info("Transitioning RM to Standby mode"); - adminService.transitionToStandby(true); - return; - } catch (Exception e) { - LOG.error("Failed to transition RM to Standby mode."); - } + if (rmContext.isHAEnabled()) { + try { + // Transition to standby and reinit active services + LOG.info("Transitioning RM to Standby mode"); + rm.transitionToStandby(true); + return; + } catch (Exception e) { + LOG.error("Failed to transition RM to Standby mode."); } } } @@ -826,10 +835,6 @@ public class ResourceManager extends CompositeService implements Recoverable { webApp = builder.start(new RMWebApp(this)); } - void setConf(Configuration configuration) { - conf = configuration; - } - /** * Helper method to create and init {@link #activeServices}. This creates an * instance of {@link RMActiveServices} and initializes it. @@ -870,6 +875,39 @@ public class ResourceManager extends CompositeService implements Recoverable { return activeServices != null && activeServices.isInState(STATE.STARTED); } + synchronized void transitionToActive() throws Exception { + if (rmContext.getHAServiceState() == + HAServiceProtocol.HAServiceState.ACTIVE) { + LOG.info("Already in active state"); + return; + } + + LOG.info("Transitioning to active state"); + startActiveServices(); + rmContext.setHAServiceState(HAServiceProtocol.HAServiceState.ACTIVE); + LOG.info("Transitioned to active state"); + } + + synchronized void transitionToStandby(boolean initialize) + throws Exception { + if (rmContext.getHAServiceState() == + HAServiceProtocol.HAServiceState.STANDBY) { + LOG.info("Already in standby state"); + return; + } + + LOG.info("Transitioning to standby state"); + if (rmContext.getHAServiceState() == + HAServiceProtocol.HAServiceState.ACTIVE) { + stopActiveServices(); + if (initialize) { + createAndInitActiveServices(); + } + } + rmContext.setHAServiceState(HAServiceProtocol.HAServiceState.STANDBY); + LOG.info("Transitioned to standby state"); + } + @Override protected void serviceStart() throws Exception { try { @@ -877,6 +915,13 @@ public class ResourceManager extends CompositeService implements Recoverable { } catch(IOException ie) { throw new YarnRuntimeException("Failed to login", ie); } + + if (this.rmContext.isHAEnabled()) { + transitionToStandby(true); + } else { + transitionToActive(); + } + super.serviceStart(); } @@ -888,6 +933,8 @@ public class ResourceManager extends CompositeService implements Recoverable { @Override protected void serviceStop() throws Exception { super.serviceStop(); + transitionToStandby(false); + rmContext.setHAServiceState(HAServiceState.STOPPING); } protected ResourceTrackerService createResourceTrackerService() { From f3db30ccd8825faf3d6916d9574ca6a44f4b1433 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Wed, 11 Dec 2013 17:50:26 +0000 Subject: [PATCH 58/76] YARN-1491. Upgrade JUnit3 TestCase to JUnit 4 (Chen He via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550204 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 ++ .../util/TestLinuxResourceCalculatorPlugin.java | 16 ++++++++-------- .../yarn/util/TestWindowsBasedProcessTree.java | 6 +++--- .../TestWindowsResourceCalculatorPlugin.java | 10 +++++----- .../hadoop/yarn/util/TestYarnVersionInfo.java | 8 ++++---- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7edd4bba467..5b14f631514 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -157,6 +157,8 @@ Release 2.4.0 - UNRELEASED YARN-1481. Move internal services logic from AdminService to ResourceManager. (vinodkv via kasha) + YARN-1491. Upgrade JUnit3 TestCase to JUnit 4 (Chen He via jeagles) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLinuxResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLinuxResourceCalculatorPlugin.java index ce9fedd2338..c9a33d0f290 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLinuxResourceCalculatorPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLinuxResourceCalculatorPlugin.java @@ -23,16 +23,16 @@ import java.io.FileWriter; import java.io.IOException; import java.util.Random; -import junit.framework.TestCase; import org.apache.hadoop.fs.Path; import org.junit.Test; +import static org.junit.Assert.assertEquals; /** * A JUnit test to test {@link LinuxResourceCalculatorPlugin} * Create the fake /proc/ information and verify the parsing and calculation */ -public class TestLinuxResourceCalculatorPlugin extends TestCase { +public class TestLinuxResourceCalculatorPlugin { /** * LinuxResourceCalculatorPlugin with a fake timer */ @@ -145,7 +145,7 @@ public class TestLinuxResourceCalculatorPlugin extends TestCase { * @throws IOException */ @Test - public void testParsingProcStatAndCpuFile() throws IOException { + public void parsingProcStatAndCpuFile() throws IOException { // Write fake /proc/cpuinfo file. long numProcessors = 8; long cpuFrequencyKHz = 2392781; @@ -171,7 +171,7 @@ public class TestLinuxResourceCalculatorPlugin extends TestCase { updateStatFile(uTime, nTime, sTime); assertEquals(plugin.getCumulativeCpuTime(), FAKE_JIFFY_LENGTH * (uTime + nTime + sTime)); - assertEquals(plugin.getCpuUsage(), (float)(LinuxResourceCalculatorPlugin.UNAVAILABLE)); + assertEquals(plugin.getCpuUsage(), (float)(LinuxResourceCalculatorPlugin.UNAVAILABLE),0.0); // Advance the time and sample again to test the CPU usage calculation uTime += 100L; @@ -179,13 +179,13 @@ public class TestLinuxResourceCalculatorPlugin extends TestCase { updateStatFile(uTime, nTime, sTime); assertEquals(plugin.getCumulativeCpuTime(), FAKE_JIFFY_LENGTH * (uTime + nTime + sTime)); - assertEquals(plugin.getCpuUsage(), 6.25F); + assertEquals(plugin.getCpuUsage(), 6.25F, 0.0); // Advance the time and sample again. This time, we call getCpuUsage() only. uTime += 600L; plugin.advanceTime(300L); updateStatFile(uTime, nTime, sTime); - assertEquals(plugin.getCpuUsage(), 25F); + assertEquals(plugin.getCpuUsage(), 25F, 0.0); // Advance very short period of time (one jiffy length). // In this case, CPU usage should not be updated. @@ -194,7 +194,7 @@ public class TestLinuxResourceCalculatorPlugin extends TestCase { updateStatFile(uTime, nTime, sTime); assertEquals(plugin.getCumulativeCpuTime(), FAKE_JIFFY_LENGTH * (uTime + nTime + sTime)); - assertEquals(plugin.getCpuUsage(), 25F); // CPU usage is not updated. + assertEquals(plugin.getCpuUsage(), 25F, 0.0); // CPU usage is not updated. } /** @@ -212,7 +212,7 @@ public class TestLinuxResourceCalculatorPlugin extends TestCase { * @throws IOException */ @Test - public void testParsingProcMemFile() throws IOException { + public void parsingProcMemFile() throws IOException { long memTotal = 4058864L; long memFree = 99632L; long inactive = 567732L; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java index ef1ee39c498..d5b5c378302 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java @@ -22,10 +22,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.util.Shell; -import junit.framework.TestCase; import org.junit.Test; +import static org.junit.Assert.assertTrue; -public class TestWindowsBasedProcessTree extends TestCase { +public class TestWindowsBasedProcessTree { private static final Log LOG = LogFactory .getLog(TestWindowsBasedProcessTree.class); @@ -41,7 +41,7 @@ public class TestWindowsBasedProcessTree extends TestCase { } @Test (timeout = 30000) - public void testTree() { + public void tree() { if( !Shell.WINDOWS) { LOG.info("Platform not Windows. Not testing"); return; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java index 70dde323180..a9e20bc0644 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java @@ -18,10 +18,10 @@ package org.apache.hadoop.yarn.util; -import junit.framework.TestCase; import org.junit.Test; +import static org.junit.Assert.assertTrue; -public class TestWindowsResourceCalculatorPlugin extends TestCase { +public class TestWindowsResourceCalculatorPlugin { class WindowsResourceCalculatorPluginTester extends WindowsResourceCalculatorPlugin { @@ -33,7 +33,7 @@ public class TestWindowsResourceCalculatorPlugin extends TestCase { } @Test (timeout = 30000) - public void testParseSystemInfoString() { + public void parseSystemInfoString() { WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester(); // info str derived from windows shell command has \r\n termination tester.infoStr = "17177038848,8589467648,15232745472,6400417792,1,2805000,6261812\r\n"; @@ -51,7 +51,7 @@ public class TestWindowsResourceCalculatorPlugin extends TestCase { } @Test (timeout = 20000) - public void testRefreshAndCpuUsage() throws InterruptedException { + public void refreshAndCpuUsage() throws InterruptedException { WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester(); // info str derived from windows shell command has \r\n termination tester.infoStr = "17177038848,8589467648,15232745472,6400417792,1,2805000,6261812\r\n"; @@ -75,7 +75,7 @@ public class TestWindowsResourceCalculatorPlugin extends TestCase { } @Test (timeout = 20000) - public void testErrorInGetSystemInfo() { + public void errorInGetSystemInfo() { WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester(); // info str derived from windows shell command has \r\n termination tester.infoStr = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java index 61795cae351..47ee8220071 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java @@ -18,23 +18,23 @@ package org.apache.hadoop.yarn.util; -import junit.framework.TestCase; import java.io.IOException; import org.apache.hadoop.yarn.util.YarnVersionInfo; import org.junit.Test; - +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; /** * A JUnit test to test {@link YarnVersionInfo} */ -public class TestYarnVersionInfo extends TestCase { +public class TestYarnVersionInfo { /** * Test the yarn version info routines. * @throws IOException */ @Test - public void testVersionInfoGenerated() throws IOException { + public void versionInfoGenerated() throws IOException { // can't easily know what the correct values are going to be so just // make sure they aren't Unknown From 002840f60557eb629420a93df7b7ce18a77523b8 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Wed, 11 Dec 2013 18:58:30 +0000 Subject: [PATCH 59/76] HADOOP-10087. UserGroupInformation.getGroupNames() fails to return primary group first when JniBasedUnixGroupsMappingWithFallback is used (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550229 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../apache/hadoop/security/hadoop_user_info.c | 47 ++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 84cc137d394..1ccc28e3ba1 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -538,6 +538,9 @@ Release 2.3.0 - UNRELEASED HADOOP-10081. Client.setupIOStreams can leak socket resources on exception or error (Tsuyoshi OZAWA via jlowe) + HADOOP-10087. UserGroupInformation.getGroupNames() fails to return primary + group first when JniBasedUnixGroupsMappingWithFallback is used (cmccabe) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/hadoop_user_info.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/hadoop_user_info.c index 2b11032e1c3..5c58b96c39e 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/hadoop_user_info.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/hadoop_user_info.c @@ -122,13 +122,43 @@ int hadoop_user_info_fetch(struct hadoop_user_info *uinfo, } } +static int put_primary_gid_first(struct hadoop_user_info *uinfo) +{ + int i, num_gids = uinfo->num_gids; + gid_t first_gid; + gid_t gid; + gid_t primary = uinfo->pwd.pw_gid; + + if (num_gids < 1) { + // There are no gids, but we expected at least one. + return EINVAL; + } + first_gid = uinfo->gids[0]; + if (first_gid == primary) { + // First gid is already the primary. + return 0; + } + for (i = 1; i < num_gids; i++) { + gid = uinfo->gids[i]; + if (gid == primary) { + // swap first gid and this gid. + uinfo->gids[0] = gid; + uinfo->gids[i] = first_gid; + return 0; + } + } + // Did not find the primary gid in the list. + return EINVAL; +} + int hadoop_user_info_getgroups(struct hadoop_user_info *uinfo) { int ret, ngroups; gid_t *ngids; if (!uinfo->pwd.pw_name) { - return EINVAL; // invalid user info + // invalid user info + return EINVAL; } uinfo->num_gids = 0; if (!uinfo->gids) { @@ -141,8 +171,12 @@ int hadoop_user_info_getgroups(struct hadoop_user_info *uinfo) ngroups = uinfo->gids_size; ret = getgrouplist(uinfo->pwd.pw_name, uinfo->pwd.pw_gid, uinfo->gids, &ngroups); - if (ret != -1) { + if (ret > 0) { uinfo->num_gids = ngroups; + ret = put_primary_gid_first(uinfo); + if (ret) { + return ret; + } return 0; } ngids = realloc(uinfo->gids, sizeof(uinfo->gids[0]) * ngroups); @@ -153,11 +187,12 @@ int hadoop_user_info_getgroups(struct hadoop_user_info *uinfo) uinfo->gids_size = ngroups; ret = getgrouplist(uinfo->pwd.pw_name, uinfo->pwd.pw_gid, uinfo->gids, &ngroups); - if (ret != -1) { - uinfo->num_gids = ngroups; - return 0; + if (ret < 0) { + return EIO; } - return EIO; + uinfo->num_gids = ngroups; + ret = put_primary_gid_first(uinfo); + return ret; } #ifdef USER_TESTING From 4ff1a8203b86115ef237224624fd8650e926e739 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 11 Dec 2013 20:14:30 +0000 Subject: [PATCH 60/76] YARN-408. Change CapacityScheduler to not disable delay-scheduling by default. Contributed by Mayank Bansal. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550245 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../conf/capacity-scheduler.xml | 6 +++--- .../hadoop/yarn/server/resourcemanager/TestRM.java | 6 +++++- .../scheduler/capacity/TestLeafQueue.java | 9 ++++++--- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5b14f631514..9efcf9440bc 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -159,6 +159,9 @@ Release 2.4.0 - UNRELEASED YARN-1491. Upgrade JUnit3 TestCase to JUnit 4 (Chen He via jeagles) + YARN-408. Change CapacityScheduler to not disable delay-scheduling by default. + (Mayank Bansal via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml index 80a9fec72c4..2bed464bcc6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml @@ -99,12 +99,12 @@ yarn.scheduler.capacity.node-locality-delay - -1 + 40 Number of missed scheduling opportunities after which the CapacityScheduler attempts to schedule rack-local containers. - Typically this should be set to number of racks in the cluster, this - feature is disabled by default, set to -1. + Typically this should be set to number of nodes in the cluster, By default is setting + approximately number of nodes in one rack which is 40. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java index 0908b2fae86..205846a11de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java @@ -22,6 +22,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import javax.security.auth.login.Configuration; + import junit.framework.Assert; import org.apache.commons.logging.Log; @@ -88,7 +90,9 @@ public class TestRM { public void testAppOnMultiNode() throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); - MockRM rm = new MockRM(); + YarnConfiguration conf = new YarnConfiguration(); + conf.set("yarn.scheduler.capacity.node-locality-delay", "-1"); + MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 5120); MockNM nm2 = rm.registerNode("h2:5678", 10240); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 4f4bf2f4708..0ecb230f153 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -1066,6 +1066,9 @@ public class TestLeafQueue { assertEquals(2*GB, a.getMetrics().getAllocatedMB()); // node_1 heartbeats in and gets the DEFAULT_RACK request for app_1 + // We do not need locality delay here + doReturn(-1).when(a).getNodeLocalityDelay(); + a.assignContainers(clusterResource, node_1); assertEquals(10*GB, a.getUsedResources().getMemory()); assertEquals(2*GB, app_0.getCurrentConsumption().getMemory()); @@ -1649,7 +1652,7 @@ public class TestLeafQueue { LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E)); // before reinitialization - assertEquals(0, e.getNodeLocalityDelay()); + assertEquals(40, e.getNodeLocalityDelay()); csConf.setInt(CapacitySchedulerConfiguration .NODE_LOCALITY_DELAY, 60); @@ -1932,10 +1935,10 @@ public class TestLeafQueue { // Now, should allocate since RR(rack_1) = relax: true a.assignContainers(clusterResource, node_1_1); - verify(app_0).allocate(eq(NodeType.RACK_LOCAL), eq(node_1_1), + verify(app_0,never()).allocate(eq(NodeType.RACK_LOCAL), eq(node_1_1), any(Priority.class), any(ResourceRequest.class), any(Container.class)); assertEquals(0, app_0.getSchedulingOpportunities(priority)); - assertEquals(0, app_0.getTotalRequiredResources(priority)); + assertEquals(1, app_0.getTotalRequiredResources(priority)); // Now sanity-check node_local app_0_requests_0.add( From ed4d318d681eaa59bf4e9048c13175260da5a719 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Wed, 11 Dec 2013 21:16:48 +0000 Subject: [PATCH 61/76] HDFS-5023. TestSnapshotPathINodes.testAllowSnapshot is failing with jdk7 (Mit Desai via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550261 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../namenode/TestSnapshotPathINodes.java | 21 +++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6090ef78e4c..2a97b9a11d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -587,6 +587,9 @@ Release 2.4.0 - UNRELEASED HDFS-5633. Improve OfflineImageViewer to use less memory. (jing9) + HDFS-5023. TestSnapshotPathINodes.testAllowSnapshot is failing with jdk7 + (Mit Desai via jeagles) + OPTIMIZATIONS HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java index fbfbf679f05..5c5ff443c4b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapsho import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -57,7 +58,7 @@ public class TestSnapshotPathINodes { static private DistributedFileSystem hdfs; @BeforeClass - static public void setUp() throws Exception { + public static void setUp() throws Exception { conf = new Configuration(); cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(REPLICATION) @@ -68,12 +69,16 @@ public class TestSnapshotPathINodes { fsdir = fsn.getFSDirectory(); hdfs = cluster.getFileSystem(); + } + + @Before + public void reset() throws Exception { DFSTestUtil.createFile(hdfs, file1, 1024, REPLICATION, seed); DFSTestUtil.createFile(hdfs, file2, 1024, REPLICATION, seed); } @AfterClass - static public void tearDown() throws Exception { + public static void tearDown() throws Exception { if (cluster != null) { cluster.shutdown(); } @@ -251,6 +256,8 @@ public class TestSnapshotPathINodes { System.out.println("The exception is expected: " + fnfe); } } + hdfs.deleteSnapshot(sub1, "s1"); + hdfs.disallowSnapshot(sub1); } /** @@ -308,6 +315,8 @@ public class TestSnapshotPathINodes { sub1.toString()); assertEquals(inodes[components.length - 3].getFullPathName(), dir.toString()); + hdfs.deleteSnapshot(sub1, "s2"); + hdfs.disallowSnapshot(sub1); } static private Snapshot s4; @@ -367,6 +376,8 @@ public class TestSnapshotPathINodes { sub1.toString()); assertEquals(inodes[components.length - 3].getFullPathName(), dir.toString()); + hdfs.deleteSnapshot(sub1, "s4"); + hdfs.disallowSnapshot(sub1); } /** @@ -375,9 +386,6 @@ public class TestSnapshotPathINodes { */ @Test (timeout=15000) public void testSnapshotPathINodesAfterModification() throws Exception { - //file1 was deleted, create it again. - DFSTestUtil.createFile(hdfs, file1, 1024, REPLICATION, seed); - // First check the INode for /TestSnapshot/sub1/file1 String[] names = INode.getPathNames(file1.toString()); byte[][] components = INode.getPathComponents(names); @@ -385,7 +393,6 @@ public class TestSnapshotPathINodes { INode[] inodes = nodesInPath.getINodes(); // The number of inodes should be equal to components.length assertEquals(inodes.length, components.length); - assertSnapshot(nodesInPath, false, s4, -1); // The last INode should be associated with file1 assertEquals(inodes[components.length - 1].getFullPathName(), @@ -434,5 +441,7 @@ public class TestSnapshotPathINodes { assertEquals(newInodes[last].getFullPathName(), file1.toString()); // The modification time of the INode for file3 should have been changed Assert.assertFalse(modTime == newInodes[last].getModificationTime()); + hdfs.deleteSnapshot(sub1, "s3"); + hdfs.disallowSnapshot(sub1); } } From 9678020e59cf073f74cce70ac57d1f6869349a36 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Wed, 11 Dec 2013 21:30:16 +0000 Subject: [PATCH 62/76] HDFS-4201. NPE in BPServiceActor#sendHeartBeat (jxiang via cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550269 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hdfs/server/datanode/BPOfferService.java | 16 +++++-- .../server/datanode/TestBPOfferService.java | 48 +++++++++++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2a97b9a11d8..ee9b964724c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -813,6 +813,8 @@ Release 2.3.0 - UNRELEASED HDFS-5074. Allow starting up from an fsimage checkpoint in the middle of a segment. (Todd Lipcon via atm) + HDFS-4201. NPE in BPServiceActor#sendHeartBeat. (jxiang via cmccabe) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index 65da792048e..e646be9a650 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -274,12 +274,22 @@ class BPOfferService { synchronized void verifyAndSetNamespaceInfo(NamespaceInfo nsInfo) throws IOException { if (this.bpNSInfo == null) { this.bpNSInfo = nsInfo; - + boolean success = false; + // Now that we know the namespace ID, etc, we can pass this to the DN. // The DN can now initialize its local storage if we are the // first BP to handshake, etc. - dn.initBlockPool(this); - return; + try { + dn.initBlockPool(this); + success = true; + } finally { + if (!success) { + // The datanode failed to initialize the BP. We need to reset + // the namespace info so that other BPService actors still have + // a chance to set it, and re-initialize the datanode. + this.bpNSInfo = null; + } + } } else { checkNSEquality(bpNSInfo.getBlockPoolID(), nsInfo.getBlockPoolID(), "Blockpool ID"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java index 420dfca2c8a..c5cb6c77f12 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java @@ -25,7 +25,9 @@ import static org.junit.Assert.assertSame; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -294,6 +296,47 @@ public class TestBPOfferService { } } + /** + * Test datanode block pool initialization error handling. + * Failure in initializing a block pool should not cause NPE. + */ + @Test + public void testBPInitErrorHandling() throws Exception { + final DataNode mockDn = Mockito.mock(DataNode.class); + Mockito.doReturn(true).when(mockDn).shouldRun(); + Configuration conf = new Configuration(); + File dnDataDir = new File( + new File(TEST_BUILD_DATA, "testBPInitErrorHandling"), "data"); + conf.set(DFS_DATANODE_DATA_DIR_KEY, dnDataDir.toURI().toString()); + Mockito.doReturn(conf).when(mockDn).getConf(); + Mockito.doReturn(new DNConf(conf)).when(mockDn).getDnConf(); + Mockito.doReturn(DataNodeMetrics.create(conf, "fake dn")). + when(mockDn).getMetrics(); + final AtomicInteger count = new AtomicInteger(); + Mockito.doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + if (count.getAndIncrement() == 0) { + throw new IOException("faked initBlockPool exception"); + } + // The initBlockPool is called again. Now mock init is done. + Mockito.doReturn(mockFSDataset).when(mockDn).getFSDataset(); + return null; + } + }).when(mockDn).initBlockPool(Mockito.any(BPOfferService.class)); + BPOfferService bpos = setupBPOSForNNs(mockDn, mockNN1, mockNN2); + bpos.start(); + try { + waitForInitialization(bpos); + List actors = bpos.getBPServiceActors(); + assertEquals(1, actors.size()); + BPServiceActor actor = actors.get(0); + waitForBlockReport(actor.getNameNodeProxy()); + } finally { + bpos.stop(); + } + } + private void waitForOneToFail(final BPOfferService bpos) throws Exception { GenericTestUtils.waitFor(new Supplier() { @@ -311,6 +354,11 @@ public class TestBPOfferService { */ private BPOfferService setupBPOSForNNs( DatanodeProtocolClientSideTranslatorPB ... nns) throws IOException { + return setupBPOSForNNs(mockDn, nns); + } + + private BPOfferService setupBPOSForNNs(DataNode mockDn, + DatanodeProtocolClientSideTranslatorPB ... nns) throws IOException { // Set up some fake InetAddresses, then override the connectToNN // function to return the corresponding proxies. From 8373c4134afef0b7fb08a63b44ef0131faacec00 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Thu, 12 Dec 2013 03:04:43 +0000 Subject: [PATCH 63/76] HDFS-5637. Try to refeatchToken while local read InvalidToken occurred. (Liang Xie via junping_du) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550335 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/hdfs/DFSInputStream.java | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ee9b964724c..0997058f974 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -590,6 +590,9 @@ Release 2.4.0 - UNRELEASED HDFS-5023. TestSnapshotPathINodes.testAllowSnapshot is failing with jdk7 (Mit Desai via jeagles) + HDFS-5637. Try to refeatchToken while local read InvalidToken occurred. + (Liang Xie via junping_du) + OPTIMIZATIONS HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 25441ff2b4c..df252343758 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -65,6 +65,7 @@ import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.unix.DomainSocket; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.IdentityHashStore; @@ -590,7 +591,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, // The encryption key used is invalid. refetchEncryptionKey--; dfsClient.clearDataEncryptionKey(); - } else if (ex instanceof InvalidBlockTokenException && refetchToken > 0) { + } else if ((ex instanceof InvalidBlockTokenException || ex instanceof InvalidToken) + && refetchToken > 0) { DFSClient.LOG.info("Will fetch a new access token and retry, " + "access token was invalid when connecting to " + targetAddr + " : " + ex); @@ -976,7 +978,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, // The encryption key used is invalid. refetchEncryptionKey--; dfsClient.clearDataEncryptionKey(); - } else if (e instanceof InvalidBlockTokenException && refetchToken > 0) { + } else if ((e instanceof InvalidBlockTokenException || e instanceof InvalidToken) + && refetchToken > 0) { DFSClient.LOG.info("Will get a new access token and retry, " + "access token was invalid when connecting to " + targetAddr + " : " + e); From fba994ffe20d387e8ed875e727fc3d93f7097101 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 12 Dec 2013 07:01:24 +0000 Subject: [PATCH 64/76] HDFS-5648. Get rid of FsDatasetImpl#perVolumeReplicaMap. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1550357 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 2 + .../fsdataset/impl/FsDatasetImpl.java | 110 +++++++----------- .../datanode/fsdataset/impl/FsVolumeList.java | 9 +- .../hdfs/server/protocol/DatanodeStorage.java | 19 +++ 4 files changed, 64 insertions(+), 76 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt index f822f9cf33f..1391c14f82e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt @@ -123,3 +123,5 @@ IMPROVEMENTS: HDFS-5484. StorageType and State in DatanodeStorageInfo in NameNode is not accurate. (Eric Sirianni via Arpit Agarwal) + HDFS-5648. Get rid of FsDatasetImpl#perVolumeReplicaMap. (Arpit Agarwal) + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 806e9f70402..b81235625e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -38,6 +38,7 @@ import javax.management.NotCompliantMBeanException; import javax.management.ObjectName; import javax.management.StandardMBean; +import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -197,15 +198,8 @@ class FsDatasetImpl implements FsDatasetSpi { final FsDatasetCache cacheManager; private final int validVolsRequired; - // TODO HDFS-2832: Consider removing duplicated block info from these - // two maps and move the perVolumeReplicaMap to FsVolumeImpl. - // This might require some refactoring. final ReplicaMap volumeMap; - // Map from StorageID to ReplicaMap. - final Map perVolumeReplicaMap; - - // Used for synchronizing access to usage stats private final Object statsLock = new Object(); @@ -254,7 +248,6 @@ class FsDatasetImpl implements FsDatasetSpi { LOG.info("Added volume - " + dir + ", StorageType: " + storageType); } volumeMap = new ReplicaMap(this); - perVolumeReplicaMap = new HashMap(); @SuppressWarnings("unchecked") final VolumeChoosingPolicy blockChooserImpl = @@ -263,7 +256,7 @@ class FsDatasetImpl implements FsDatasetSpi { RoundRobinVolumeChoosingPolicy.class, VolumeChoosingPolicy.class), conf); volumes = new FsVolumeList(volArray, volsFailed, blockChooserImpl); - volumes.initializeReplicaMaps(perVolumeReplicaMap, volumeMap, this); + volumes.initializeReplicaMaps(volumeMap); File[] roots = new File[storage.getNumStorageDirs()]; for (int idx = 0; idx < storage.getNumStorageDirs(); idx++) { @@ -661,7 +654,6 @@ class FsDatasetImpl implements FsDatasetSpi { // Replace finalized replica by a RBW replica in replicas map volumeMap.add(bpid, newReplicaInfo); - perVolumeReplicaMap.get(v.getStorageID()).add(bpid, newReplicaInfo); return newReplicaInfo; } @@ -792,7 +784,6 @@ class FsDatasetImpl implements FsDatasetSpi { ReplicaBeingWritten newReplicaInfo = new ReplicaBeingWritten(b.getBlockId(), b.getGenerationStamp(), v, f.getParentFile()); volumeMap.add(b.getBlockPoolId(), newReplicaInfo); - perVolumeReplicaMap.get(v.getStorageID()).add(b.getBlockPoolId(), newReplicaInfo); return newReplicaInfo; } @@ -911,7 +902,6 @@ class FsDatasetImpl implements FsDatasetSpi { rbw.setBytesAcked(visible); // overwrite the RBW in the volume map volumeMap.add(b.getBlockPoolId(), rbw); - perVolumeReplicaMap.get(v.getStorageID()).add(b.getBlockPoolId(), rbw); return rbw; } @@ -931,7 +921,6 @@ class FsDatasetImpl implements FsDatasetSpi { ReplicaInPipeline newReplicaInfo = new ReplicaInPipeline(b.getBlockId(), b.getGenerationStamp(), v, f.getParentFile()); volumeMap.add(b.getBlockPoolId(), newReplicaInfo); - perVolumeReplicaMap.get(v.getStorageID()).add(b.getBlockPoolId(), newReplicaInfo); return newReplicaInfo; } @@ -1000,8 +989,6 @@ class FsDatasetImpl implements FsDatasetSpi { newReplicaInfo = new FinalizedReplica(replicaInfo, v, dest.getParentFile()); } volumeMap.add(bpid, newReplicaInfo); - perVolumeReplicaMap.get(newReplicaInfo.getVolume().getStorageID()) - .add(bpid, newReplicaInfo); return newReplicaInfo; } @@ -1015,8 +1002,6 @@ class FsDatasetImpl implements FsDatasetSpi { if (replicaInfo != null && replicaInfo.getState() == ReplicaState.TEMPORARY) { // remove from volumeMap volumeMap.remove(b.getBlockPoolId(), b.getLocalBlock()); - perVolumeReplicaMap.get(replicaInfo.getVolume().getStorageID()) - .remove(b.getBlockPoolId(), b.getLocalBlock()); // delete the on-disk temp file if (delBlockFromDisk(replicaInfo.getBlockFile(), @@ -1051,39 +1036,6 @@ class FsDatasetImpl implements FsDatasetSpi { return true; } - private BlockListAsLongs getBlockReportWithReplicaMap( - String bpid, ReplicaMap rMap) { - int size = rMap.size(bpid); - ArrayList finalized = new ArrayList(size); - ArrayList uc = new ArrayList(); - if (size == 0) { - return new BlockListAsLongs(finalized, uc); - } - - synchronized(this) { - for (ReplicaInfo b : rMap.replicas(bpid)) { - switch(b.getState()) { - case FINALIZED: - finalized.add(b); - break; - case RBW: - case RWR: - uc.add(b); - break; - case RUR: - ReplicaUnderRecovery rur = (ReplicaUnderRecovery)b; - uc.add(rur.getOriginalReplica()); - break; - case TEMPORARY: - break; - default: - assert false : "Illegal ReplicaInfo state."; - } - } - return new BlockListAsLongs(finalized, uc); - } - } - @Override // FsDatasetSpi public List getCacheReport(String bpid) { return cacheManager.getCachedBlocks(bpid); @@ -1091,16 +1043,49 @@ class FsDatasetImpl implements FsDatasetSpi { @Override public Map getBlockReports(String bpid) { - Map blockReportMap = + Map blockReportsMap = new HashMap(); - for (FsVolumeImpl v : getVolumes()) { - ReplicaMap rMap = perVolumeReplicaMap.get(v.getStorageID()); - BlockListAsLongs blockList = getBlockReportWithReplicaMap(bpid, rMap); - blockReportMap.put(v.toDatanodeStorage(), blockList); + Map> finalized = + new HashMap>(); + Map> uc = + new HashMap>(); + + for (FsVolumeSpi v : volumes.volumes) { + finalized.put(v.getStorageID(), new ArrayList()); + uc.put(v.getStorageID(), new ArrayList()); } - return blockReportMap; + synchronized(this) { + for (ReplicaInfo b : volumeMap.replicas(bpid)) { + switch(b.getState()) { + case FINALIZED: + finalized.get(b.getVolume().getStorageID()).add(b); + break; + case RBW: + case RWR: + uc.get(b.getVolume().getStorageID()).add(b); + break; + case RUR: + ReplicaUnderRecovery rur = (ReplicaUnderRecovery)b; + uc.get(rur.getVolume().getStorageID()).add(rur.getOriginalReplica()); + break; + case TEMPORARY: + break; + default: + assert false : "Illegal ReplicaInfo state."; + } + } + } + + for (FsVolumeImpl v : volumes.volumes) { + ArrayList finalizedList = finalized.get(v.getStorageID()); + ArrayList ucList = uc.get(v.getStorageID()); + blockReportsMap.put(v.toDatanodeStorage(), + new BlockListAsLongs(finalizedList, ucList)); + } + + return blockReportsMap; } /** @@ -1244,7 +1229,6 @@ class FsDatasetImpl implements FsDatasetSpi { v.clearPath(bpid, parent); } volumeMap.remove(bpid, invalidBlks[i]); - perVolumeReplicaMap.get(v.getStorageID()).remove(bpid, invalidBlks[i]); } // If the block is cached, start uncaching it. cacheManager.uncacheBlock(bpid, invalidBlks[i].getBlockId()); @@ -1374,8 +1358,6 @@ class FsDatasetImpl implements FsDatasetSpi { LOG.warn("Removing replica " + bpid + ":" + b.getBlockId() + " on failed volume " + fv.getCurrentDir().getAbsolutePath()); ib.remove(); - perVolumeReplicaMap.get(fv.getStorageID()) - .remove(bpid, b.getBlockId()); removedBlocks++; } } @@ -1492,7 +1474,6 @@ class FsDatasetImpl implements FsDatasetSpi { // Block is in memory and not on the disk // Remove the block from volumeMap volumeMap.remove(bpid, blockId); - perVolumeReplicaMap.get(vol.getStorageID()).remove(bpid, blockId); final DataBlockScanner blockScanner = datanode.getBlockScanner(); if (blockScanner != null) { blockScanner.deleteBlock(bpid, new Block(blockId)); @@ -1516,8 +1497,6 @@ class FsDatasetImpl implements FsDatasetSpi { ReplicaInfo diskBlockInfo = new FinalizedReplica(blockId, diskFile.length(), diskGS, vol, diskFile.getParentFile()); volumeMap.add(bpid, diskBlockInfo); - perVolumeReplicaMap.get(vol.getStorageID()) - .remove(bpid, diskBlockInfo); final DataBlockScanner blockScanner = datanode.getBlockScanner(); if (blockScanner != null) { blockScanner.addBlock(new ExtendedBlock(bpid, diskBlockInfo)); @@ -1792,13 +1771,6 @@ class FsDatasetImpl implements FsDatasetSpi { volumes.addBlockPool(bpid, conf); volumeMap.initBlockPool(bpid); volumes.getAllVolumesMap(bpid, volumeMap); - - // TODO: Avoid the double scan. - for (FsVolumeImpl v : getVolumes()) { - ReplicaMap rMap = perVolumeReplicaMap.get(v.getStorageID()); - rMap.initBlockPool(bpid); - volumes.getVolumeMap(bpid, v, rMap); - } } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index 708ffaf06e6..89830fcf249 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -90,14 +90,9 @@ class FsVolumeList { return remaining; } - void initializeReplicaMaps(Map perVolumeReplicaMap, - ReplicaMap globalReplicaMap, - Object mutex) throws IOException { + void initializeReplicaMaps(ReplicaMap globalReplicaMap) throws IOException { for (FsVolumeImpl v : volumes) { - ReplicaMap rMap = new ReplicaMap(mutex); - v.getVolumeMap(rMap); - perVolumeReplicaMap.put(v.getStorageID(), rMap); - globalReplicaMap.addAll(rMap); + v.getVolumeMap(globalReplicaMap); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java index da80f21e985..271f71091d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorage.java @@ -71,4 +71,23 @@ public class DatanodeStorage { public static String generateUuid() { return "DS-" + UUID.randomUUID(); } + + @Override + public boolean equals(Object other){ + if (other == this) { + return true; + } + + if ((other == null) || + !(other instanceof DatanodeStorage)) { + return false; + } + DatanodeStorage otherStorage = (DatanodeStorage) other; + return otherStorage.getStorageID().compareTo(getStorageID()) == 0; + } + + @Override + public int hashCode() { + return getStorageID().hashCode(); + } } From 93511259ba1c3df222b6542ad2269f292eb5f442 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 12 Dec 2013 07:07:05 +0000 Subject: [PATCH 65/76] Update CHANGES.txt in preparation for merging branch HDFS-2832 to trunk git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2832@1550361 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 129 ++++++++++++++++++ .../hadoop-hdfs/CHANGES_HDFS-2832.txt | 127 ----------------- 2 files changed, 129 insertions(+), 127 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0997058f974..73399348f79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -13,6 +13,10 @@ Trunk (Unreleased) HDFS-3125. Add JournalService to enable Journal Daemon. (suresh) + HDFS-2832. Heterogeneous Storages support in HDFS phase 1 - treat DataNode + as a collection of storages (see breakdown of tasks below for features and + contributors). + IMPROVEMENTS HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common. @@ -440,6 +444,131 @@ Trunk (Unreleased) HDFS-5626. dfsadmin -report shows incorrect cache values. (cmccabe) + BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS + + HDFS-4985. Add storage type to the protocol and expose it in block report + and block locations. (Arpit Agarwal) + + HDFS-5115. Make StorageID a UUID. (Arpit Agarwal) + + HDFS-5000. DataNode configuration should allow specifying storage type. + (Arpit Agarwal) + + HDFS-4987. Namenode changes to track multiple storages per datanode. + (szetszwo) + + HDFS-5154. Fix TestBlockManager and TestDatanodeDescriptor after HDFS-4987. + (Junping Du via szetszwo) + + HDFS-5009. Include storage information in the LocatedBlock. (szetszwo) + + HDFS-5134. Move blockContentsStale, heartbeatedSinceFailover and + firstBlockReport from DatanodeDescriptor to DatanodeStorageInfo; and + fix a synchronization problem in DatanodeStorageInfo. (szetszwo) + + HDFS-5157. Add StorageType to FsVolume. (Junping Du via szetszwo) + + HDFS-4990. Change BlockPlacementPolicy to choose storages instead of + datanodes. (szetszwo) + + HDFS-5232. Protocol changes to transmit StorageUuid. (Arpit Agarwal) + + HDFS-5233. Use Datanode UUID to identify Datanodes. (Arpit Agarwal) + + HDFS-5222. Move block schedule information from DatanodeDescriptor to + DatanodeStorageInfo. (szetszwo) + + HDFS-4988. Datanode must support all the volumes as individual storages. + (Arpit Agarwal) + + HDFS-5377. Heartbeats from Datandode should include one storage report + per storage directory. (Arpit Agarwal) + + HDFS-5398. NameNode changes to process storage reports per storage + directory. (Arpit Agarwal) + + HDFS-5390. Send one incremental block report per storage directory. + (Arpit Agarwal) + + HDFS-5401. Fix NPE in Directory Scanner. (Arpit Agarwal) + + HDFS-5417. Fix storage IDs in PBHelper and UpgradeUtilities. (szetszwo) + + HDFS-5214. Fix NPEs in BlockManager and DirectoryScanner. (Arpit Agarwal) + + HDFS-5435. File append fails to initialize storageIDs. (Junping Du via + Arpit Agarwal) + + HDFS-5437. Fix TestBlockReport and TestBPOfferService failures. (Arpit + Agarwal) + + HDFS-5447. Fix TestJspHelper. (Arpit Agarwal) + + HDFS-5452. Fix TestReplicationPolicy and TestBlocksScheduledCounter. + + HDFS-5448. Datanode should generate its ID on first registration. (Arpit + Agarwal) + + HDFS-5448. Fix break caused by previous checkin for HDFS-5448. (Arpit + Agarwal) + + HDFS-5455. NN should update storageMap on first heartbeat. (Arpit Agarwal) + + HDFS-5457. Fix TestDatanodeRegistration, TestFsck and TestAddBlockRetry. + (Contributed by szetszwo) + + HDFS-5466. Update storage IDs when the pipeline is updated. (Contributed + by szetszwo) + + HDFS-5439. Fix TestPendingReplication. (Contributed by Junping Du, Arpit + Agarwal) + + HDFS-5470. Add back trunk's reportDiff algorithm to the branch. + (Contributed by szetszwo) + + HDFS-5472. Fix TestDatanodeManager, TestSafeMode and + TestNNThroughputBenchmark (Contributed by szetszwo) + + HDFS-5475. NN incorrectly tracks more than one replica per DN. (Arpit + Agarwal) + + HDFS-5481. Fix TestDataNodeVolumeFailure in branch HDFS-2832. (Contributed + by Junping Du) + + HDFS-5480. Update Balancer for HDFS-2832. (Contributed by szetszwo) + + HDFS-5486. Fix TestNameNodeMetrics for HDFS-2832. (Arpit Agarwal) + + HDFS-5491. Update editsStored for HDFS-2832. (Arpit Agarwal) + + HDFS-5494. Fix findbugs warnings for HDFS-2832. (Arpit Agarwal) + + HDFS-5508. Fix compilation error after merge. (Contributed by szetszwo) + + HDFS-5501. Fix pendingReceivedRequests tracking in BPServiceActor. (Arpit + Agarwal) + + HDFS-5510. Fix a findbug warning in DataStorage.java on HDFS-2832 branch. + (Junping Du via Arpit Agarwal) + + HDFS-5515. Fix TestDFSStartupVersions for HDFS-2832. (Arpit Agarwal) + + HDFS-5527. Fix TestUnderReplicatedBlocks on branch HDFS-2832. (Arpit + Agarwal) + + HDFS-5547. Fix build break after merge from trunk to HDFS-2832. (Arpit + Agarwal) + + HDFS-5542. Fix TODO and clean up the code in HDFS-2832. (Contributed by + szetszwo) + + HDFS-5559. Fix TestDatanodeConfig in HDFS-2832. (Contributed by szetszwo) + + HDFS-5484. StorageType and State in DatanodeStorageInfo in NameNode is + not accurate. (Eric Sirianni via Arpit Agarwal) + + HDFS-5648. Get rid of FsDatasetImpl#perVolumeReplicaMap. (Arpit Agarwal) + Release 2.4.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt deleted file mode 100644 index 1391c14f82e..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES_HDFS-2832.txt +++ /dev/null @@ -1,127 +0,0 @@ - BREAKDOWN OF HDFS-2832 ENABLE SUPPORT FOR HETEROGENEOUS STORAGES IN HDFS - -IMPROVEMENTS: - - HDFS-4985. Add storage type to the protocol and expose it in block report - and block locations. (Arpit Agarwal) - - HDFS-5115. Make StorageID a UUID. (Arpit Agarwal) - - HDFS-5000. DataNode configuration should allow specifying storage type. - (Arpit Agarwal) - - HDFS-4987. Namenode changes to track multiple storages per datanode. - (szetszwo) - - HDFS-5154. Fix TestBlockManager and TestDatanodeDescriptor after HDFS-4987. - (Junping Du via szetszwo) - - HDFS-5009. Include storage information in the LocatedBlock. (szetszwo) - - HDFS-5134. Move blockContentsStale, heartbeatedSinceFailover and - firstBlockReport from DatanodeDescriptor to DatanodeStorageInfo; and - fix a synchronization problem in DatanodeStorageInfo. (szetszwo) - - HDFS-5157. Add StorageType to FsVolume. (Junping Du via szetszwo) - - HDFS-4990. Change BlockPlacementPolicy to choose storages instead of - datanodes. (szetszwo) - - HDFS-5232. Protocol changes to transmit StorageUuid. (Arpit Agarwal) - - HDFS-5233. Use Datanode UUID to identify Datanodes. (Arpit Agarwal) - - HDFS-5222. Move block schedule information from DatanodeDescriptor to - DatanodeStorageInfo. (szetszwo) - - HDFS-4988. Datanode must support all the volumes as individual storages. - (Arpit Agarwal) - - HDFS-5377. Heartbeats from Datandode should include one storage report - per storage directory. (Arpit Agarwal) - - HDFS-5398. NameNode changes to process storage reports per storage - directory. (Arpit Agarwal) - - HDFS-5390. Send one incremental block report per storage directory. - (Arpit Agarwal) - - HDFS-5401. Fix NPE in Directory Scanner. (Arpit Agarwal) - - HDFS-5417. Fix storage IDs in PBHelper and UpgradeUtilities. (szetszwo) - - HDFS-5214. Fix NPEs in BlockManager and DirectoryScanner. (Arpit Agarwal) - - HDFS-5435. File append fails to initialize storageIDs. (Junping Du via - Arpit Agarwal) - - HDFS-5437. Fix TestBlockReport and TestBPOfferService failures. (Arpit - Agarwal) - - HDFS-5447. Fix TestJspHelper. (Arpit Agarwal) - - HDFS-5452. Fix TestReplicationPolicy and TestBlocksScheduledCounter. - - HDFS-5448. Datanode should generate its ID on first registration. (Arpit - Agarwal) - - HDFS-5448. Fix break caused by previous checkin for HDFS-5448. (Arpit - Agarwal) - - HDFS-5455. NN should update storageMap on first heartbeat. (Arpit Agarwal) - - HDFS-5457. Fix TestDatanodeRegistration, TestFsck and TestAddBlockRetry. - (Contributed by szetszwo) - - HDFS-5466. Update storage IDs when the pipeline is updated. (Contributed - by szetszwo) - - HDFS-5439. Fix TestPendingReplication. (Contributed by Junping Du, Arpit - Agarwal) - - HDFS-5470. Add back trunk's reportDiff algorithm to the branch. - (Contributed by szetszwo) - - HDFS-5472. Fix TestDatanodeManager, TestSafeMode and - TestNNThroughputBenchmark (Contributed by szetszwo) - - HDFS-5475. NN incorrectly tracks more than one replica per DN. (Arpit - Agarwal) - - HDFS-5481. Fix TestDataNodeVolumeFailure in branch HDFS-2832. (Contributed - by Junping Du) - - HDFS-5480. Update Balancer for HDFS-2832. (Contributed by szetszwo) - - HDFS-5486. Fix TestNameNodeMetrics for HDFS-2832. (Arpit Agarwal) - - HDFS-5491. Update editsStored for HDFS-2832. (Arpit Agarwal) - - HDFS-5494. Fix findbugs warnings for HDFS-2832. (Arpit Agarwal) - - HDFS-5508. Fix compilation error after merge. (Contributed by szetszwo) - - HDFS-5501. Fix pendingReceivedRequests tracking in BPServiceActor. (Arpit - Agarwal) - - HDFS-5510. Fix a findbug warning in DataStorage.java on HDFS-2832 branch. - (Junping Du via Arpit Agarwal) - - HDFS-5515. Fix TestDFSStartupVersions for HDFS-2832. (Arpit Agarwal) - - HDFS-5527. Fix TestUnderReplicatedBlocks on branch HDFS-2832. (Arpit - Agarwal) - - HDFS-5547. Fix build break after merge from trunk to HDFS-2832. (Arpit - Agarwal) - - HDFS-5542. Fix TODO and clean up the code in HDFS-2832. (Contributed by - szetszwo) - - HDFS-5559. Fix TestDatanodeConfig in HDFS-2832. (Contributed by szetszwo) - - HDFS-5484. StorageType and State in DatanodeStorageInfo in NameNode is - not accurate. (Eric Sirianni via Arpit Agarwal) - - HDFS-5648. Get rid of FsDatasetImpl#perVolumeReplicaMap. (Arpit Agarwal) - From 329093ae1bed3e9cfd20fa425bdc60d88a536e51 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 12 Dec 2013 07:47:09 +0000 Subject: [PATCH 66/76] HDFS-2832: Update binary file editsStored for TestOfflineEditsViewer git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550364 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/test/resources/editsStored | Bin 4592 -> 4627 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored index 3f37c932bc0e7a88bb56bef94c11f60fd5648ea1..3c7fcbbf9809c28b49a88f037af110511ca54464 100644 GIT binary patch literal 4627 zcmcIodrTBZ82|Q`$KCM;Me$Kr6zl7R0~7&UR^CDd(H5;HoC_R?930&{K$@CpOl`D{ zHmUz;TN_h?##BSp+WLsqXfpGJj- zY>BsR><)cBk*Y!fd65kelUP&du-mFsyW+OnnuYw-6e|b-LiVwo>s>yH6%3k|QBi6Y zwN-JLUCm8)wN`aDHwo=9JvVV@ew&_m2%!g;@3ibRX*2+VRDxe4J!m>J7AyEO zQe9NjjeOHu5Fa&!#G}F4Vy`B^o(Qg~rhSg>g@g>*4II5#uAaGf_jes4HNr@wEJ9>r zb)9Ojay!*FTSGm1r{u3N$qoBJ*poxgCQ~reNRX2CA9NuEt(iv-R~(pohG~b49M1T4 zbHW6w9XfItOs+d`dgu%jL{pp`e(^Q*F8i% z2lZfur6rg$M|0;0LmL)Xo}zUNG2A&|Y_zwQwl>;P5uY9xbSaD?j^WP+VNT{3u2skr zO_uj|UJ0ifV>zkThQvEVv`KqckEa=hSUv$~ZN7m}P@;KzP+ZUXcD80<95F-=^ddVO zs}#4*t*EtjH`^#ZWi$;);`Z`uWnx?LGN+`G0cgjAj7*qRQ2hQB+8IZrc(=E`L!v)m z0@n>~OT_xHL%a)s0{f4p-(Zy}lD6*9u;KLi8BZ8IUDsl7dd87%20xh#uSEmH?=TPo z0F1U(7(&FDtZ={KoCKt4k(U9)%_~++V`Nl{eXX5mwlXq_OQ!9tNMA>WGo{grKH7qi z-sNn$O~u1nW6O=3XA~WL*5G{Org`RN9M*?kaU_YOy8d3;PtO2SD`J5u0H$Fn0|cC1@>?KF-gGi3 z_UCmr&t}O>(0jo06M#yjpOnSP+qa4&q6{;zl+m5wnP*oILw=PEiUT1TZEXyBrXE>~ z0a6Xn$`4@^AYUCk?wXS$;cnDs;fD+mbgIt$4dbEl7w=WQcWoErnWN`nb)e(%R$h>g z@nog(F{`|}m{Dub$>|N`fL3(FbLu4{s+y1J6^D+vr|Tb}{Xx3^d;Hqi|Mj+uWF~-x zyvV^ihrQRt8|YNQ24aaDF@_KPVu~)cJ1y>zq>%y4!%{|fumce*@3557o31!m(SLF& zOIecM16FjwsOSbs(Pa*{{3u~W)U5a+0~m`hU-B^IWZa1&eSb%N^j?OXqDR(ZAS0u+ zqC?mO$X5cPo}6tG?nW&YKV$$?%iBkQ@yy4ai08rQVWsC7&jLLUs{2%YF*N345 zS1GE)?Q9eR394`wfdqfg1$6%`e?50za5xhVO;Ya>=`M+ZWX z8Hxab7>56)Q1H{-l~?2G)WuANi(dFyQ~oarbV~Lxo4B)=guincrIoZCB4iW9C9YG{ zYKz0Jxa=160f@dk)i#UU?($wPZ``6aha_$U#G8_r@D*OvbnkDoq>%ySVkrZdr#zCJ zY;(#ZgW|(K_uo9sHm7_)&fs#!ltOCCBX!X1 zj3qx{$tcemKhX~>ME}sRl_(AN5@)r&yyy5)ws{s&67XX*-hlCvzWtb!JYO&fQV4T_ UpVY~bA8w-GAy9m+C4Q86^p*DQHoqR(N{-)?u1Ka3=E7;5E^6{6WBzy#JrTE)bhy(xis|+EKH5f zjE&6W4H-BMjm-57fY6XZKy3xPl*A9Gb8G8LGgT%(i2>?kOIF!@Vd5tp1E6uU@fgch zTw0J?lwOoyTEMuZC`j@6n5@LqDuKoQe4)1%-NFG&d9iVBXaUP9yefWzx)|%yxBSMY&bDFh}W`y?u0AM z5Cg9;V+q;+Shd@~f5dcF2;wmYD+ZW8ul%HtqQvCv)FK98aRD@02uK~A8zIC9Obd)o zi?$mGnf0t@7KUoH25O80ml9lrbviYyO!_4P)oDYFl{qXfem8vq zBUTr z_xNQdR|xX4%sCv#5=p7cqFBU$&SJ0E#dc A2><{9 From 5fd7230671bb3d90081d4138c364c3f65b35cdbc Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Thu, 12 Dec 2013 18:26:40 +0000 Subject: [PATCH 67/76] HDFS-5647. Merge INodeDirectory.Feature and INodeFile.Feature. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550469 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../namenode/DirectoryWithQuotaFeature.java | 2 +- .../FileUnderConstructionFeature.java | 2 +- .../hadoop/hdfs/server/namenode/INode.java | 43 +----------------- .../hdfs/server/namenode/INodeDirectory.java | 29 +----------- .../hdfs/server/namenode/INodeFile.java | 33 ++------------ .../namenode/INodeWithAdditionalFields.java | 44 +++++++++++++++++++ .../snapshot/FileWithSnapshotFeature.java | 2 +- .../hdfs/server/namenode/TestINodeFile.java | 5 ++- 9 files changed, 60 insertions(+), 103 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 73399348f79..de710e6ebef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -246,6 +246,9 @@ Trunk (Unreleased) HDFS-5629. Support HTTPS in JournalNode and SecondaryNameNode. (Haohui Mai via jing9) + HDFS-5647. Merge INodeDirectory.Feature and INodeFile.Feature. (Haohui Mai + via jing9) + OPTIMIZATIONS HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java index c03a7971d11..511b5fdf6b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException; /** * Quota feature for {@link INodeDirectory}. */ -public final class DirectoryWithQuotaFeature extends INodeDirectory.Feature { +public final class DirectoryWithQuotaFeature implements INode.Feature { public static final long DEFAULT_NAMESPACE_QUOTA = Long.MAX_VALUE; public static final long DEFAULT_DISKSPACE_QUOTA = HdfsConstants.QUOTA_RESET; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java index 0a7371dc485..782a859e047 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; * Feature for under-construction file. */ @InterfaceAudience.Private -public class FileUnderConstructionFeature extends INodeFile.Feature { +public class FileUnderConstructionFeature implements INode.Feature { private String clientName; // lease holder private final String clientMachine; // if client is a cluster node too. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index 1ba48a012e0..c90369e4a5b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -756,47 +756,6 @@ public abstract class INode implements INodeAttributes, Diff.Element { /** INode feature such as {@link FileUnderConstructionFeature} * and {@link DirectoryWithQuotaFeature}. */ - interface Feature> { - /** @return the next feature. */ - public F getNextFeature(); - - /** Set the next feature. */ - public void setNextFeature(F next); - - /** Utility methods such as addFeature and removeFeature. */ - static class Util { - /** - * Add a feature to the linked list. - * @return the new head. - */ - static > F addFeature(F feature, F head) { - feature.setNextFeature(head); - return feature; - } - - /** - * Remove a feature from the linked list. - * @return the new head. - */ - static > F removeFeature(F feature, F head) { - if (feature == head) { - final F newHead = head.getNextFeature(); - head.setNextFeature(null); - return newHead; - } else if (head != null) { - F prev = head; - F curr = head.getNextFeature(); - for (; curr != null && curr != feature; - prev = curr, curr = curr.getNextFeature()) - ; - if (curr != null) { - prev.setNextFeature(curr.getNextFeature()); - curr.setNextFeature(null); - return head; - } - } - throw new IllegalStateException("Feature " + feature + " not found."); - } - } + public interface Feature { } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index 05e98673658..71fdc28d810 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -45,20 +45,6 @@ import com.google.common.base.Preconditions; */ public class INodeDirectory extends INodeWithAdditionalFields implements INodeDirectoryAttributes { - /** Directory related features such as quota and snapshots. */ - public static abstract class Feature implements INode.Feature { - private Feature nextFeature; - - @Override - public Feature getNextFeature() { - return nextFeature; - } - - @Override - public void setNextFeature(Feature next) { - this.nextFeature = next; - } - } /** Cast INode to INodeDirectory. */ public static INodeDirectory valueOf(INode inode, Object path @@ -78,9 +64,6 @@ public class INodeDirectory extends INodeWithAdditionalFields private List children = null; - /** A linked list of {@link Feature}s. */ - private Feature headFeature = null; - /** constructor */ public INodeDirectory(long id, byte[] name, PermissionStatus permissions, long mtime) { @@ -102,7 +85,7 @@ public class INodeDirectory extends INodeWithAdditionalFields } } if (copyFeatures) { - this.headFeature = other.headFeature; + this.features = other.features; } } @@ -160,7 +143,7 @@ public class INodeDirectory extends INodeWithAdditionalFields * otherwise, return null. */ public final DirectoryWithQuotaFeature getDirectoryWithQuotaFeature() { - for(Feature f = headFeature; f != null; f = f.nextFeature) { + for (Feature f : features) { if (f instanceof DirectoryWithQuotaFeature) { return (DirectoryWithQuotaFeature)f; } @@ -182,14 +165,6 @@ public class INodeDirectory extends INodeWithAdditionalFields return quota; } - private void addFeature(Feature f) { - headFeature = INode.Feature.Util.addFeature(f, headFeature); - } - - private void removeFeature(Feature f) { - headFeature = INode.Feature.Util.removeFeature(f, headFeature); - } - private int searchChildren(byte[] name) { return children == null? -1: Collections.binarySearch(children, name); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index cced4e7bd30..1d420e84761 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -41,23 +41,6 @@ import com.google.common.base.Preconditions; @InterfaceAudience.Private public class INodeFile extends INodeWithAdditionalFields implements INodeFileAttributes, BlockCollection { - /** - * A feature contains specific information for a type of INodeFile. E.g., - * we can have separate features for Under-Construction and Snapshot. - */ - public static abstract class Feature implements INode.Feature { - private Feature nextFeature; - - @Override - public Feature getNextFeature() { - return nextFeature; - } - - @Override - public void setNextFeature(Feature next) { - this.nextFeature = next; - } - } /** The same as valueOf(inode, path, false). */ public static INodeFile valueOf(INode inode, String path @@ -120,8 +103,6 @@ public class INodeFile extends INodeWithAdditionalFields private BlockInfo[] blocks; - private Feature headFeature; - INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime, long atime, BlockInfo[] blklist, short replication, long preferredBlockSize) { @@ -135,7 +116,7 @@ public class INodeFile extends INodeWithAdditionalFields super(that); this.header = that.header; this.blocks = that.blocks; - this.headFeature = that.headFeature; + this.features = that.features; } public INodeFile(INodeFile that, FileDiffList diffs) { @@ -144,14 +125,6 @@ public class INodeFile extends INodeWithAdditionalFields this.addSnapshotFeature(diffs); } - private void addFeature(Feature f) { - headFeature = INode.Feature.Util.addFeature(f, headFeature); - } - - private void removeFeature(Feature f) { - headFeature = INode.Feature.Util.removeFeature(f, headFeature); - } - /** @return true unconditionally. */ @Override public final boolean isFile() { @@ -171,7 +144,7 @@ public class INodeFile extends INodeWithAdditionalFields * otherwise, return null. */ public final FileUnderConstructionFeature getFileUnderConstructionFeature() { - for (Feature f = this.headFeature; f != null; f = f.nextFeature) { + for (Feature f : features) { if (f instanceof FileUnderConstructionFeature) { return (FileUnderConstructionFeature) f; } @@ -283,7 +256,7 @@ public class INodeFile extends INodeWithAdditionalFields * otherwise, return null. */ public final FileWithSnapshotFeature getFileWithSnapshotFeature() { - for (Feature f = headFeature; f != null; f = f.nextFeature) { + for (Feature f: features) { if (f instanceof FileWithSnapshotFeature) { return (FileWithSnapshotFeature) f; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java index f8b63ec2265..bc74ad2a10b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java @@ -95,6 +95,9 @@ public abstract class INodeWithAdditionalFields extends INode /** For implementing {@link LinkedElement}. */ private LinkedElement next = null; + /** An array {@link Feature}s. */ + private static final Feature[] EMPTY_FEATURE = new Feature[0]; + protected Feature[] features = EMPTY_FEATURE; private INodeWithAdditionalFields(INode parent, long id, byte[] name, long permission, long modificationTime, long accessTime) { @@ -262,4 +265,45 @@ public abstract class INodeWithAdditionalFields extends INode public final void setAccessTime(long accessTime) { this.accessTime = accessTime; } + + protected void addFeature(Feature f) { + int size = features.length; + Feature[] arr = new Feature[size + 1]; + if (size != 0) { + System.arraycopy(features, 0, arr, 0, size); + } + arr[size] = f; + features = arr; + } + + protected void removeFeature(Feature f) { + int size = features.length; + Preconditions.checkState(size > 0, "Feature " + + f.getClass().getSimpleName() + " not found."); + + if (size == 1) { + Preconditions.checkState(features[0] == f, "Feature " + + f.getClass().getSimpleName() + " not found."); + features = EMPTY_FEATURE; + return; + } + + Feature[] arr = new Feature[size - 1]; + int j = 0; + boolean overflow = false; + for (Feature f1 : features) { + if (f1 != f) { + if (j == size - 1) { + overflow = true; + break; + } else { + arr[j++] = f1; + } + } + } + + Preconditions.checkState(!overflow && j == size - 1, "Feature " + + f.getClass().getSimpleName() + " not found."); + features = arr; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java index 737603a45bd..b74d2694789 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hdfs.server.namenode.Quota; * Feature for file with snapshot-related information. */ @InterfaceAudience.Private -public class FileWithSnapshotFeature extends INodeFile.Feature { +public class FileWithSnapshotFeature implements INode.Feature { private final FileDiffList diffs; private boolean isCurrentFileDeleted = false; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java index 2f921907927..642599d09fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.Time; import org.junit.Test; import org.mockito.Mockito; @@ -921,6 +922,7 @@ public class TestINodeFile { public void testDotdotInodePath() throws Exception { final Configuration conf = new Configuration(); MiniDFSCluster cluster = null; + DFSClient client = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); @@ -933,7 +935,7 @@ public class TestINodeFile { long parentId = fsdir.getINode("/").getId(); String testPath = "/.reserved/.inodes/" + dirId + "/.."; - DFSClient client = new DFSClient(NameNode.getAddress(conf), conf); + client = new DFSClient(NameNode.getAddress(conf), conf); HdfsFileStatus status = client.getFileInfo(testPath); assertTrue(parentId == status.getFileId()); @@ -943,6 +945,7 @@ public class TestINodeFile { assertTrue(parentId == status.getFileId()); } finally { + IOUtils.cleanup(LOG, client); if (cluster != null) { cluster.shutdown(); } From ee0530ee0e595464087820cacafc4ab4227cf2a6 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Thu, 12 Dec 2013 18:30:12 +0000 Subject: [PATCH 68/76] MAPREDUCE-5674. Missing start and finish time in mapred.JobStatus. Contributed by Chuan Liu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550472 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapreduce/TypeConverter.java | 3 +++ .../hadoop/mapreduce/TestTypeConverter.java | 26 +++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 7476c79bb17..0969b272bfd 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -290,6 +290,9 @@ Release 2.3.0 - UNRELEASED event: TA_TOO_MANY_FETCH_FAILURE at KILLED for TaskAttemptImpl (Gera Shegalov via jlowe) + MAPREDUCE-5674. Missing start and finish time in mapred.JobStatus. + (Chuan Liu via cnauroth) + Release 2.2.0 - 2013-10-13 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java index a28b5eb6eb1..d62045ba2ca 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java @@ -300,6 +300,8 @@ public class TypeConverter { .getCleanupProgress(), fromYarn(jobreport.getJobState()), jobPriority, jobreport.getUser(), jobreport.getJobName(), jobreport .getJobFile(), trackingUrl, jobreport.isUber()); + jobStatus.setStartTime(jobreport.getStartTime()); + jobStatus.setFinishTime(jobreport.getFinishTime()); jobStatus.setFailureInfo(jobreport.getDiagnostics()); return jobStatus; } @@ -441,6 +443,7 @@ public class TypeConverter { ); jobStatus.setSchedulingInfo(trackingUrl); // Set AM tracking url jobStatus.setStartTime(application.getStartTime()); + jobStatus.setFinishTime(application.getFinishTime()); jobStatus.setFailureInfo(application.getDiagnostics()); jobStatus.setNeededMem(application.getApplicationResourceUsageReport().getNeededResources().getMemory()); jobStatus.setNumReservedSlots(application.getApplicationResourceUsageReport().getNumReservedContainers()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java index 16a72a87afb..ea3324e5023 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java @@ -27,6 +27,8 @@ import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.JobStatus.State; +import org.apache.hadoop.mapreduce.v2.api.records.JobId; +import org.apache.hadoop.mapreduce.v2.api.records.JobReport; import org.apache.hadoop.mapreduce.v2.api.records.JobState; import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; @@ -71,6 +73,7 @@ public class TestTypeConverter { @Test public void testFromYarn() throws Exception { int appStartTime = 612354; + int appFinishTime = 612355; YarnApplicationState state = YarnApplicationState.RUNNING; ApplicationId applicationId = ApplicationId.newInstance(0, 0); ApplicationReport applicationReport = Records @@ -78,6 +81,7 @@ public class TestTypeConverter { applicationReport.setApplicationId(applicationId); applicationReport.setYarnApplicationState(state); applicationReport.setStartTime(appStartTime); + applicationReport.setFinishTime(appFinishTime); applicationReport.setUser("TestTypeConverter-user"); ApplicationResourceUsageReport appUsageRpt = Records .newRecord(ApplicationResourceUsageReport.class); @@ -91,6 +95,7 @@ public class TestTypeConverter { applicationReport.setApplicationResourceUsageReport(appUsageRpt); JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile"); Assert.assertEquals(appStartTime, jobStatus.getStartTime()); + Assert.assertEquals(appFinishTime, jobStatus.getFinishTime()); Assert.assertEquals(state.toString(), jobStatus.getState().toString()); } @@ -172,4 +177,25 @@ public class TestTypeConverter { Assert.assertEquals("QueueInfo children weren't properly converted", returned.getQueueChildren().size(), 1); } + + @Test + public void testFromYarnJobReport() throws Exception { + int jobStartTime = 612354; + int jobFinishTime = 612355; + JobState state = JobState.RUNNING; + JobId jobId = Records.newRecord(JobId.class); + JobReport jobReport = Records.newRecord(JobReport.class); + ApplicationId applicationId = ApplicationId.newInstance(0, 0); + jobId.setAppId(applicationId); + jobId.setId(0); + jobReport.setJobId(jobId); + jobReport.setJobState(state); + jobReport.setStartTime(jobStartTime); + jobReport.setFinishTime(jobFinishTime); + jobReport.setUser("TestTypeConverter-user"); + JobStatus jobStatus = TypeConverter.fromYarn(jobReport, "dummy-jobfile"); + Assert.assertEquals(jobStartTime, jobStatus.getStartTime()); + Assert.assertEquals(jobFinishTime, jobStatus.getFinishTime()); + Assert.assertEquals(state.toString(), jobStatus.getState().toString()); + } } From a4819e70dbf88b0905a6669078afa1ff0924ad4f Mon Sep 17 00:00:00 2001 From: Sanjay Radia Date: Thu, 12 Dec 2013 18:56:45 +0000 Subject: [PATCH 69/76] HADOOP-10044 Improve the javadoc of rpc code (sanjay Radia) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550486 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../org/apache/hadoop/ipc/RpcConstants.java | 14 ++ .../java/org/apache/hadoop/ipc/Server.java | 127 +++++++++++++++--- 3 files changed, 121 insertions(+), 22 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 1ccc28e3ba1..b37d7dee81a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -280,6 +280,8 @@ Trunk (Unreleased) HDFS-5471. CacheAdmin -listPools fails when user lacks permissions to view all pools (Andrew Wang via Colin Patrick McCabe) + HADOOP-10044 Improve the javadoc of rpc code (sanjay Radia) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java index 831862bb237..c457500e902 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcConstants.java @@ -37,10 +37,24 @@ public class RpcConstants { public static final int INVALID_RETRY_COUNT = -1; + /** + * The Rpc-connection header is as follows + * +----------------------------------+ + * | "hrpc" 4 bytes | + * +----------------------------------+ + * | Version (1 byte) | + * +----------------------------------+ + * | Service Class (1 byte) | + * +----------------------------------+ + * | AuthProtocol (1 byte) | + * +----------------------------------+ + */ + /** * The first four bytes of Hadoop RPC connections */ public static final ByteBuffer HEADER = ByteBuffer.wrap("hrpc".getBytes()); + public static final int HEADER_LEN_AFTER_HRPC_PART = 3; // 3 bytes that follow // 1 : Introduce ping and server does not throw away RPCs // 3 : Introduce the protocol into the RPC connection header diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 7fb395cdb05..7f569408902 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -1105,6 +1105,9 @@ public abstract class Server { this.channel = channel; this.lastContact = lastContact; this.data = null; + + // the buffer is initialized to read the "hrpc" and after that to read + // the length of the Rpc-packet (i.e 4 bytes) this.dataLengthBuffer = ByteBuffer.allocate(4); this.unwrappedData = null; this.unwrappedDataLengthBuffer = ByteBuffer.allocate(4); @@ -1200,7 +1203,16 @@ public abstract class Server { } } - private Throwable getCauseForInvalidToken(IOException e) { + /** + * Some exceptions ({@link RetriableException} and {@link StandbyException}) + * that are wrapped as a cause of parameter e are unwrapped so that they can + * be sent as the true cause to the client side. In case of + * {@link InvalidToken} we go one level deeper to get the true cause. + * + * @param e the exception that may have a cause we want to unwrap. + * @return the true cause for some exceptions. + */ + private Throwable getTrueCause(IOException e) { Throwable cause = e; while (cause != null) { if (cause instanceof RetriableException) { @@ -1223,6 +1235,18 @@ public abstract class Server { return e; } + /** + * Process saslMessage and send saslResponse back + * @param saslMessage received SASL message + * @throws WrappedRpcServerException setup failed due to SASL negotiation + * failure, premature or invalid connection context, or other state + * errors. This exception needs to be sent to the client. This + * exception will wrap {@link RetriableException}, + * {@link InvalidToken}, {@link StandbyException} or + * {@link SaslException}. + * @throws IOException if sending reply fails + * @throws InterruptedException + */ private void saslProcess(RpcSaslProto saslMessage) throws WrappedRpcServerException, IOException, InterruptedException { if (saslContextEstablished) { @@ -1239,7 +1263,7 @@ public abstract class Server { // attempting user could be null AUDITLOG.warn(AUTH_FAILED_FOR + this.toString() + ":" + attemptingUser + " (" + e.getLocalizedMessage() + ")"); - throw (IOException) getCauseForInvalidToken(e); + throw (IOException) getTrueCause(e); } if (saslServer != null && saslServer.isComplete()) { @@ -1274,13 +1298,26 @@ public abstract class Server { } } + /** + * Process a saslMessge. + * @param saslMessage received SASL message + * @return the sasl response to send back to client + * @throws SaslException if authentication or generating response fails, + * or SASL protocol mixup + * @throws IOException if a SaslServer cannot be created + * @throws AccessControlException if the requested authentication type + * is not supported or trying to re-attempt negotiation. + * @throws InterruptedException + */ private RpcSaslProto processSaslMessage(RpcSaslProto saslMessage) - throws IOException, InterruptedException { + throws SaslException, IOException, AccessControlException, + InterruptedException { RpcSaslProto saslResponse = null; final SaslState state = saslMessage.getState(); // required switch (state) { case NEGOTIATE: { if (sentNegotiate) { + // FIXME shouldn't this be SaslException? throw new AccessControlException( "Client already attempted negotiation"); } @@ -1402,12 +1439,30 @@ public abstract class Server { } } + /** + * This method reads in a non-blocking fashion from the channel: + * this method is called repeatedly when data is present in the channel; + * when it has enough data to process one rpc it processes that rpc. + * + * On the first pass, it processes the connectionHeader, + * connectionContext (an outOfBand RPC) and at most one RPC request that + * follows that. On future passes it will process at most one RPC request. + * + * Quirky things: dataLengthBuffer (4 bytes) is used to read "hrpc" OR + * rpc request length. + * + * @return -1 in case of error, else num bytes read so far + * @throws WrappedRpcServerException - an exception that has already been + * sent back to the client that does not require verbose logging + * by the Listener thread + * @throws IOException - internal error that should not be returned to + * client, typically failure to respond to client + * @throws InterruptedException + */ public int readAndProcess() throws WrappedRpcServerException, IOException, InterruptedException { while (true) { - /* Read at most one RPC. If the header is not read completely yet - * then iterate until we read first RPC or until there is no data left. - */ + // dataLengthBuffer is used to read "hrpc" or the rpc-packet length int count = -1; if (dataLengthBuffer.remaining() > 0) { count = channelRead(channel, dataLengthBuffer); @@ -1416,9 +1471,11 @@ public abstract class Server { } if (!connectionHeaderRead) { - //Every connection is expected to send the header. + // Every connection is expected to send the header; + // so far we read "hrpc" of the connection header. if (connectionHeaderBuf == null) { - connectionHeaderBuf = ByteBuffer.allocate(3); + // for the bytes that follow "hrpc", in the connection header + connectionHeaderBuf = ByteBuffer.allocate(HEADER_LEN_AFTER_HRPC_PART); } count = channelRead(channel, connectionHeaderBuf); if (count < 0 || connectionHeaderBuf.remaining() > 0) { @@ -1451,27 +1508,30 @@ public abstract class Server { // this may switch us into SIMPLE authProtocol = initializeAuthContext(connectionHeaderBuf.get(2)); - dataLengthBuffer.clear(); + dataLengthBuffer.clear(); // clear to next read rpc packet len connectionHeaderBuf = null; connectionHeaderRead = true; - continue; + continue; // connection header read, now read 4 bytes rpc packet len } - if (data == null) { + if (data == null) { // just read 4 bytes - length of RPC packet dataLengthBuffer.flip(); dataLength = dataLengthBuffer.getInt(); checkDataLength(dataLength); + // Set buffer for reading EXACTLY the RPC-packet length and no more. data = ByteBuffer.allocate(dataLength); } - + // Now read the RPC packet count = channelRead(channel, data); if (data.remaining() == 0) { - dataLengthBuffer.clear(); + dataLengthBuffer.clear(); // to read length of future rpc packets data.flip(); boolean isHeaderRead = connectionContextRead; processOneRpc(data.array()); data = null; + // the last rpc-request we processed could have simply been the + // connectionContext; if so continue to read the first RPC. if (!isHeaderRead) { continue; } @@ -1508,8 +1568,16 @@ public abstract class Server { return authProtocol; } + /** + * Process the Sasl's Negotiate request, including the optimization of + * accelerating token negotiation. + * @return the response to Negotiate request - the list of enabled + * authMethods and challenge if the TOKENS are supported. + * @throws SaslException - if attempt to generate challenge fails. + * @throws IOException - if it fails to create the SASL server for Tokens + */ private RpcSaslProto buildSaslNegotiateResponse() - throws IOException, InterruptedException { + throws InterruptedException, SaslException, IOException { RpcSaslProto negotiateMessage = negotiateResponse; // accelerate token negotiation by sending initial challenge // in the negotiation response @@ -1635,8 +1703,11 @@ public abstract class Server { /** * Process a wrapped RPC Request - unwrap the SASL packet and process * each embedded RPC request - * @param buf - SASL wrapped request of one or more RPCs + * @param inBuf - SASL wrapped request of one or more RPCs * @throws IOException - SASL packet cannot be unwrapped + * @throws WrappedRpcServerException - an exception that has already been + * sent back to the client that does not require verbose logging + * by the Listener thread * @throws InterruptedException */ private void unwrapPacketAndProcessRpcs(byte[] inBuf) @@ -1677,13 +1748,21 @@ public abstract class Server { } /** - * Process an RPC Request - handle connection setup and decoding of - * request into a Call + * Process one RPC Request from buffer read from socket stream + * - decode rpc in a rpc-Call + * - handle out-of-band RPC requests such as the initial connectionContext + * - A successfully decoded RpcCall will be deposited in RPC-Q and + * its response will be sent later when the request is processed. + * + * Prior to this call the connectionHeader ("hrpc...") has been handled and + * if SASL then SASL has been established and the buf we are passed + * has been unwrapped from SASL. + * * @param buf - contains the RPC request header and the rpc request * @throws IOException - internal error that should not be returned to * client, typically failure to respond to client - * @throws WrappedRpcServerException - an exception to be sent back to - * the client that does not require verbose logging by the + * @throws WrappedRpcServerException - an exception that is sent back to the + * client in this method and does not require verbose logging by the * Listener thread * @throws InterruptedException */ @@ -1753,8 +1832,11 @@ public abstract class Server { } /** - * Process an RPC Request - the connection headers and context must - * have been already read + * Process an RPC Request + * - the connection headers and context must have been already read. + * - Based on the rpcKind, decode the rpcRequest. + * - A successfully decoded RpcCall will be deposited in RPC-Q and + * its response will be sent later when the request is processed. * @param header - RPC request header * @param dis - stream to request payload * @throws WrappedRpcServerException - due to fatal rpc layer issues such @@ -1803,7 +1885,8 @@ public abstract class Server { * @param dis - stream to request payload * @throws WrappedRpcServerException - setup failed due to SASL * negotiation failure, premature or invalid connection context, - * or other state errors + * or other state errors. This exception needs to be sent to the + * client. * @throws IOException - failed to send a response back to the client * @throws InterruptedException */ From 8a79be36bd097a2f0e4a84625e782a7fa1421549 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 12 Dec 2013 20:50:11 +0000 Subject: [PATCH 70/76] YARN-1325. Modified RM HA configuration validation to also ensure that multiple RMs are configured. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550524 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../org/apache/hadoop/yarn/conf/HAUtil.java | 15 +++++----- .../hadoop/yarn/conf/YarnConfiguration.java | 11 ++++--- .../apache/hadoop/yarn/conf/TestHAUtil.java | 29 ++++++++++++++----- .../yarn/server/resourcemanager/TestRMHA.java | 8 +++-- .../recovery/TestZKRMStateStore.java | 2 +- .../hadoop/yarn/server/MiniYARNCluster.java | 2 +- 7 files changed, 47 insertions(+), 23 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9efcf9440bc..ce954cf4cc4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -162,6 +162,9 @@ Release 2.4.0 - UNRELEASED YARN-408. Change CapacityScheduler to not disable delay-scheduling by default. (Mayank Bansal via vinodkv) + YARN-1325. Modified RM HA configuration validation to also ensure that + multiple RMs are configured. (Xuan Gong via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java index 3720fab57d1..1e8a7c4c78b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java @@ -60,19 +60,18 @@ public class HAUtil { throws YarnRuntimeException { verifyAndSetRMHAIds(conf); verifyAndSetRMHAId(conf); - verifyAndSetAllRpcAddresses(conf); + verifyAndSetAllServiceAddresses(conf); } private static void verifyAndSetRMHAIds(Configuration conf) { Collection ids = conf.getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS); - if (ids.size() <= 0) { + if (ids.size() < 2) { throwBadConfigurationException( getInvalidValueMessage(YarnConfiguration.RM_HA_IDS, - conf.get(YarnConfiguration.RM_HA_IDS))); - } else if (ids.size() == 1) { - LOG.warn(getRMHAIdsWarningMessage(ids.toString())); + conf.get(YarnConfiguration.RM_HA_IDS) + + "\nHA mode requires atleast two RMs")); } StringBuilder setValue = new StringBuilder(); @@ -123,8 +122,8 @@ public class HAUtil { } } - public static void verifyAndSetAllRpcAddresses(Configuration conf) { - for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) { + public static void verifyAndSetAllServiceAddresses(Configuration conf) { + for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { verifyAndSetConfValue(confKey, conf); } } @@ -176,7 +175,7 @@ public class HAUtil { @InterfaceAudience.Private @VisibleForTesting static String getConfKeyForRMInstance(String prefix, Configuration conf) { - return YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS.contains(prefix) + return YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS.contains(prefix) ? addSuffix(prefix, getRMHAId(conf)) : prefix; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 839765c04a4..12b5e2f5e77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -23,11 +23,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -285,14 +287,15 @@ public class YarnConfiguration extends Configuration { public static final String RM_HA_IDS = RM_HA_PREFIX + "rm-ids"; public static final String RM_HA_ID = RM_HA_PREFIX + "id"; - public static final List RM_RPC_ADDRESS_CONF_KEYS = + @Private + public static final List RM_SERVICES_ADDRESS_CONF_KEYS = Collections.unmodifiableList(Arrays.asList( RM_ADDRESS, RM_SCHEDULER_ADDRESS, RM_ADMIN_ADDRESS, RM_RESOURCE_TRACKER_ADDRESS, - RM_WEBAPP_ADDRESS, - RM_WEBAPP_HTTPS_ADDRESS)); + HttpConfig.isSecure() ? RM_WEBAPP_HTTPS_ADDRESS + : RM_WEBAPP_ADDRESS)); //////////////////////////////// // RM state store configs @@ -940,7 +943,7 @@ public class YarnConfiguration extends Configuration { public InetSocketAddress getSocketAddr( String name, String defaultAddress, int defaultPort) { String address; - if (HAUtil.isHAEnabled(this) && RM_RPC_ADDRESS_CONF_KEYS.contains(name)) { + if (HAUtil.isHAEnabled(this) && RM_SERVICES_ADDRESS_CONF_KEYS.contains(name)) { address = HAUtil.getConfValueForRMInstance(name, defaultAddress, this); } else { address = get(name, defaultAddress); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java index 0577006ac8f..a710c361c9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java @@ -53,7 +53,7 @@ public class TestHAUtil { conf.set(YarnConfiguration.RM_HA_IDS, RM_NODE_IDS_UNTRIMMED); conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED); - for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { // configuration key itself cannot contains space/tab/return chars. conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED); conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); @@ -94,13 +94,27 @@ public class TestHAUtil { StringUtils.getStringCollection(RM_NODE_IDS), HAUtil.getRMHAIds(conf)); assertEquals("Should be saved as Trimmed string", RM1_NODE_ID, HAUtil.getRMHAId(conf)); - for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { assertEquals("RPC address not set for " + confKey, RM1_ADDRESS, conf.get(confKey)); } conf.clear(); - conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID); + conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID); + try { + HAUtil.verifyAndSetConfiguration(conf); + } catch (YarnRuntimeException e) { + assertEquals("YarnRuntimeException by verifyAndSetRMHAIds()", + HAUtil.BAD_CONFIG_MESSAGE_PREFIX + + HAUtil.getInvalidValueMessage(YarnConfiguration.RM_HA_IDS, + conf.get(YarnConfiguration.RM_HA_IDS) + + "\nHA mode requires atleast two RMs"), + e.getMessage()); + } + + conf.clear(); + conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + "," + + RM1_NODE_ID); try { HAUtil.verifyAndSetConfiguration(conf); } catch (YarnRuntimeException e) { @@ -112,8 +126,9 @@ public class TestHAUtil { conf.clear(); conf.set(YarnConfiguration.RM_HA_ID, RM_INVALID_NODE_ID); - conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID); - for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) { + conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + "," + + RM1_NODE_ID); + for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { // simulate xml with invalid node id conf.set(confKey + RM_INVALID_NODE_ID, RM_INVALID_NODE_ID); } @@ -130,7 +145,7 @@ public class TestHAUtil { conf.clear(); // simulate the case HAUtil.RM_RPC_ADDRESS_CONF_KEYS are not set conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); - conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID); + conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); try { HAUtil.verifyAndSetConfiguration(conf); fail("Should throw YarnRuntimeException. by Configuration#set()"); @@ -147,7 +162,7 @@ public class TestHAUtil { conf.clear(); conf.set(YarnConfiguration.RM_HA_IDS, RM2_NODE_ID + "," + RM3_NODE_ID); conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED); - for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED); conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java index d1fbf1c07d4..4804423ca07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java @@ -46,13 +46,17 @@ public class TestRMHA { private static final String RM1_ADDRESS = "0.0.0.0:0"; private static final String RM1_NODE_ID = "rm1"; + private static final String RM2_ADDRESS = "1.1.1.1:1"; + private static final String RM2_NODE_ID = "rm2"; + @Before public void setUp() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); - conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID); - for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) { + conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); + for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS); + conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); } conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 924a30fd2a5..6c7c472cbdf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -142,7 +142,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName()); conf.set(YarnConfiguration.ZK_RM_STATE_STORE_ADDRESS, hostPort); conf.set(YarnConfiguration.RM_HA_ID, rmId); - for (String rpcAddress : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) { + for (String rpcAddress : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { conf.set(HAUtil.addSuffix(rpcAddress, rmId), "localhost:0"); } conf.set(HAUtil.addSuffix(YarnConfiguration.RM_ADMIN_ADDRESS, rmId), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 9829e86ab68..95b1ba5be8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -295,7 +295,7 @@ public class MiniYARNCluster extends CompositeService { String rmId = "rm" + index; String hostname = MiniYARNCluster.getHostname(); conf.set(YarnConfiguration.RM_HA_ID, rmId); - for (String confKey : YarnConfiguration.RM_RPC_ADDRESS_CONF_KEYS) { + for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) { conf.set(HAUtil.addSuffix(confKey, rmId), hostname + ":0"); } } From 288506022ff64e6413da682659459504adfbf1a8 Mon Sep 17 00:00:00 2001 From: Hitesh Shah Date: Thu, 12 Dec 2013 22:55:06 +0000 Subject: [PATCH 71/76] HADOOP-8753. LocalDirAllocator throws ArithmeticException: divide by zero when there is no available space on configured local dir. Contributed by Benoy Antony. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550570 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b37d7dee81a..449beed3e5c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -282,6 +282,9 @@ Trunk (Unreleased) HADOOP-10044 Improve the javadoc of rpc code (sanjay Radia) + HADOOP-8753. LocalDirAllocator throws "ArithmeticException: / by zero" when + there is no available space on configured local dir. (Benoy Antony via hitesh) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java index 77353be2f75..88b4d4e42c4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java @@ -365,6 +365,10 @@ public class LocalDirAllocator { totalAvailable += availableOnDisk[i]; } + if (totalAvailable == 0){ + throw new DiskErrorException("No space available in any of the local directories."); + } + // Keep rolling the wheel till we get a valid path Random r = new java.util.Random(); while (numDirsSearched < numDirs && returnPath == null) { From 38c32ed239d45051f5e4a51e3f29b201c17a4288 Mon Sep 17 00:00:00 2001 From: Jian He Date: Thu, 12 Dec 2013 23:07:17 +0000 Subject: [PATCH 72/76] YARN-1311. Fixed app specific scheduler-events' names to be app-attempt based. Contributed by Vinod Kumar Vavilapalli git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550579 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../rmapp/attempt/RMAppAttemptImpl.java | 12 ++-- .../scheduler/capacity/CapacityScheduler.java | 26 ++++---- ...ava => AppAttemptAddedSchedulerEvent.java} | 8 +-- ...a => AppAttemptRemovedSchedulerEvent.java} | 7 ++- .../scheduler/event/SchedulerEventType.java | 6 +- .../scheduler/fair/FairScheduler.java | 32 +++++----- .../scheduler/fifo/FifoScheduler.java | 41 ++++++++----- .../server/resourcemanager/Application.java | 4 +- .../resourcemanager/TestFifoScheduler.java | 14 ++--- .../resourcemanager/TestResourceManager.java | 7 ++- .../attempt/TestRMAppAttemptTransitions.java | 4 +- .../capacity/TestCapacityScheduler.java | 6 +- .../scheduler/capacity/TestLeafQueue.java | 6 +- .../scheduler/fair/TestFairScheduler.java | 60 +++++++++---------- .../scheduler/fifo/TestFifoScheduler.java | 20 +++---- 16 files changed, 138 insertions(+), 118 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/{AppAddedSchedulerEvent.java => AppAttemptAddedSchedulerEvent.java} (85%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/{AppRemovedSchedulerEvent.java => AppAttemptRemovedSchedulerEvent.java} (87%) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ce954cf4cc4..15a2d8572c6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -165,6 +165,9 @@ Release 2.4.0 - UNRELEASED YARN-1325. Modified RM HA configuration validation to also ensure that multiple RMs are configured. (Xuan Gong via vinodkv) + YARN-1311. Fixed app specific scheduler-events' names to be app-attempt + based. (vinodkv via jianhe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 67a2e414583..394465e984c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -87,8 +87,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; @@ -742,9 +742,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { new Token(id, appAttempt.rmContext.getAMRMTokenSecretManager()); - // Add the application to the scheduler + // Add the applicationAttempt to the scheduler appAttempt.eventHandler.handle( - new AppAddedSchedulerEvent(appAttempt.applicationAttemptId, + new AppAttemptAddedSchedulerEvent(appAttempt.applicationAttemptId, appAttempt.submissionContext.getQueue(), appAttempt.user)); } } @@ -1055,8 +1055,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } appAttempt.eventHandler.handle(appEvent); - appAttempt.eventHandler.handle(new AppRemovedSchedulerEvent(appAttemptId, - finalAttemptState)); + appAttempt.eventHandler.handle(new AppAttemptRemovedSchedulerEvent( + appAttemptId, finalAttemptState)); appAttempt.removeCredentials(appAttempt); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 3a2b353363a..60256398461 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -71,8 +71,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeRepo import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; @@ -417,7 +417,7 @@ public class CapacityScheduler } private synchronized void - addApplication(ApplicationAttemptId applicationAttemptId, + addApplicationAttempt(ApplicationAttemptId applicationAttemptId, String queueName, String user) { // Sanity checks @@ -466,7 +466,7 @@ public class CapacityScheduler RMAppAttemptEventType.APP_ACCEPTED)); } - private synchronized void doneApplication( + private synchronized void doneApplicationAttempt( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState) { LOG.info("Application " + applicationAttemptId + " is done." + @@ -740,18 +740,20 @@ public class CapacityScheduler nodeUpdate(nodeUpdatedEvent.getRMNode()); } break; - case APP_ADDED: + case APP_ATTEMPT_ADDED: { - AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent)event; - addApplication(appAddedEvent.getApplicationAttemptId(), appAddedEvent - .getQueue(), appAddedEvent.getUser()); + AppAttemptAddedSchedulerEvent appAttemptAddedEvent = + (AppAttemptAddedSchedulerEvent) event; + addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(), + appAttemptAddedEvent.getQueue(), appAttemptAddedEvent.getUser()); } break; - case APP_REMOVED: + case APP_ATTEMPT_REMOVED: { - AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event; - doneApplication(appRemovedEvent.getApplicationAttemptID(), - appRemovedEvent.getFinalAttemptState()); + AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = + (AppAttemptRemovedSchedulerEvent) event; + doneApplicationAttempt(appAttemptRemovedEvent.getApplicationAttemptID(), + appAttemptRemovedEvent.getFinalAttemptState()); } break; case CONTAINER_EXPIRED: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptAddedSchedulerEvent.java similarity index 85% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptAddedSchedulerEvent.java index 16211f31662..7b9ffff4b53 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptAddedSchedulerEvent.java @@ -20,15 +20,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -public class AppAddedSchedulerEvent extends SchedulerEvent { +public class AppAttemptAddedSchedulerEvent extends SchedulerEvent { private final ApplicationAttemptId applicationAttemptId; private final String queue; private final String user; - public AppAddedSchedulerEvent(ApplicationAttemptId applicationAttemptId, - String queue, String user) { - super(SchedulerEventType.APP_ADDED); + public AppAttemptAddedSchedulerEvent( + ApplicationAttemptId applicationAttemptId, String queue, String user) { + super(SchedulerEventType.APP_ATTEMPT_ADDED); this.applicationAttemptId = applicationAttemptId; this.queue = queue; this.user = user; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppRemovedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptRemovedSchedulerEvent.java similarity index 87% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppRemovedSchedulerEvent.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptRemovedSchedulerEvent.java index 963bc57b639..876d164b63a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppRemovedSchedulerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptRemovedSchedulerEvent.java @@ -21,14 +21,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; -public class AppRemovedSchedulerEvent extends SchedulerEvent { +public class AppAttemptRemovedSchedulerEvent extends SchedulerEvent { private final ApplicationAttemptId applicationAttemptId; private final RMAppAttemptState finalAttemptState; - public AppRemovedSchedulerEvent(ApplicationAttemptId applicationAttemptId, + public AppAttemptRemovedSchedulerEvent( + ApplicationAttemptId applicationAttemptId, RMAppAttemptState finalAttemptState) { - super(SchedulerEventType.APP_REMOVED); + super(SchedulerEventType.APP_ATTEMPT_REMOVED); this.applicationAttemptId = applicationAttemptId; this.finalAttemptState = finalAttemptState; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java index a78918dc4e9..dd1aec71bff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java @@ -25,9 +25,9 @@ public enum SchedulerEventType { NODE_REMOVED, NODE_UPDATE, - // Source: App - APP_ADDED, - APP_REMOVED, + // Source: RMAppAttempt + APP_ATTEMPT_ADDED, + APP_ATTEMPT_REMOVED, // Source: ContainerAllocationExpirer CONTAINER_EXPIRED diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 861fad87158..bc716c1401f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -75,8 +75,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppRepor import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; @@ -591,7 +591,7 @@ public class FairScheduler implements ResourceScheduler { * user. This will accept a new app even if the user or queue is above * configured limits, but the app will not be marked as runnable. */ - protected synchronized void addApplication( + protected synchronized void addApplicationAttempt( ApplicationAttemptId applicationAttemptId, String queueName, String user) { if (queueName == null || queueName.isEmpty()) { String message = "Reject application " + applicationAttemptId + @@ -674,7 +674,7 @@ public class FairScheduler implements ResourceScheduler { return queue; } - private synchronized void removeApplication( + private synchronized void removeApplicationAttempt( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState) { LOG.info("Application " + applicationAttemptId + " is done." + @@ -1090,22 +1090,24 @@ public class FairScheduler implements ResourceScheduler { NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event; nodeUpdate(nodeUpdatedEvent.getRMNode()); break; - case APP_ADDED: - if (!(event instanceof AppAddedSchedulerEvent)) { + case APP_ATTEMPT_ADDED: + if (!(event instanceof AppAttemptAddedSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); } - AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent)event; - String queue = appAddedEvent.getQueue(); - addApplication(appAddedEvent.getApplicationAttemptId(), queue, - appAddedEvent.getUser()); + AppAttemptAddedSchedulerEvent appAttemptAddedEvent = + (AppAttemptAddedSchedulerEvent) event; + String queue = appAttemptAddedEvent.getQueue(); + addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(), + queue, appAttemptAddedEvent.getUser()); break; - case APP_REMOVED: - if (!(event instanceof AppRemovedSchedulerEvent)) { + case APP_ATTEMPT_REMOVED: + if (!(event instanceof AppAttemptRemovedSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); } - AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event; - removeApplication(appRemovedEvent.getApplicationAttemptID(), - appRemovedEvent.getFinalAttemptState()); + AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = + (AppAttemptRemovedSchedulerEvent) event; + removeApplicationAttempt(appAttemptRemovedEvent.getApplicationAttemptID(), + appAttemptRemovedEvent.getFinalAttemptState()); break; case CONTAINER_EXPIRED: if (!(event instanceof ContainerExpiredSchedulerEvent)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 956cb492f56..c5f0bd67d92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -67,11 +66,20 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; @@ -340,7 +348,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { return nodes.get(nodeId); } - private synchronized void addApplication(ApplicationAttemptId appAttemptId, + private synchronized void addApplicationAttempt(ApplicationAttemptId appAttemptId, String user) { // TODO: Fix store FiCaSchedulerApp schedulerApp = @@ -355,7 +363,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { RMAppAttemptEventType.APP_ACCEPTED)); } - private synchronized void doneApplication( + private synchronized void doneApplicationAttempt( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState) throws IOException { @@ -689,22 +697,25 @@ public class FifoScheduler implements ResourceScheduler, Configurable { nodeUpdate(nodeUpdatedEvent.getRMNode()); } break; - case APP_ADDED: + case APP_ATTEMPT_ADDED: { - AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event; - addApplication(appAddedEvent.getApplicationAttemptId(), appAddedEvent - .getUser()); + AppAttemptAddedSchedulerEvent appAttemptAddedEvent = + (AppAttemptAddedSchedulerEvent) event; + addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(), + appAttemptAddedEvent.getUser()); } break; - case APP_REMOVED: + case APP_ATTEMPT_REMOVED: { - AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event; + AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = + (AppAttemptRemovedSchedulerEvent) event; try { - doneApplication(appRemovedEvent.getApplicationAttemptID(), - appRemovedEvent.getFinalAttemptState()); + doneApplicationAttempt( + appAttemptRemovedEvent.getApplicationAttemptID(), + appAttemptRemovedEvent.getFinalAttemptState()); } catch(IOException ie) { LOG.error("Unable to remove application " - + appRemovedEvent.getApplicationAttemptID(), ie); + + appAttemptRemovedEvent.getApplicationAttemptID(), ie); } } break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index cbce6e30d58..6767180b62a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -57,7 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.Task.State; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; @@ -166,7 +166,7 @@ public class Application { resourceManager.getClientRMService().submitApplication(request); // Notify scheduler - AppAddedSchedulerEvent appAddedEvent1 = new AppAddedSchedulerEvent( + AppAttemptAddedSchedulerEvent appAddedEvent1 = new AppAttemptAddedSchedulerEvent( this.applicationAttemptId, this.queue, this.user); scheduler.handle(appAddedEvent1); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index c57a7529a3f..2d370fcfb46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -297,8 +297,8 @@ public class TestFifoScheduler { ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1); ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId( appId1, 1); - SchedulerEvent event1 = new AppAddedSchedulerEvent(appAttemptId1, "queue", - "user"); + SchedulerEvent event1 = + new AppAttemptAddedSchedulerEvent(appAttemptId1, "queue", "user"); fs.handle(event1); List emptyId = new ArrayList(); @@ -388,15 +388,15 @@ public class TestFifoScheduler { ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1); ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId( appId1, 1); - SchedulerEvent event1 = new AppAddedSchedulerEvent(appAttemptId1, "queue", - "user"); + SchedulerEvent event1 = + new AppAttemptAddedSchedulerEvent(appAttemptId1, "queue", "user"); fs.handle(event1); ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2); ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId( appId2, 1); - SchedulerEvent event2 = new AppAddedSchedulerEvent(appAttemptId2, "queue", - "user"); + SchedulerEvent event2 = + new AppAttemptAddedSchedulerEvent(appAttemptId2, "queue", "user"); fs.handle(event2); List emptyId = new ArrayList(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index 3ed92b69664..3bcfba3abd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.util.resource.Resources; @@ -159,8 +159,9 @@ public class TestResourceManager { application.finishTask(t3); // Notify scheduler application is finished. - AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( - application.getApplicationAttemptId(), RMAppAttemptState.FINISHED); + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = + new AppAttemptRemovedSchedulerEvent( + application.getApplicationAttemptId(), RMAppAttemptState.FINISHED); resourceManager.getResourceScheduler().handle(appRemovedEvent1); checkResourceUsage(nm1, nm2); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 140b53e62ff..6c72d97080d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -85,7 +85,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; @@ -329,7 +329,7 @@ public class TestRMAppAttemptTransitions { // Check events verify(masterService). registerAppAttempt(applicationAttempt.getAppAttemptId()); - verify(scheduler).handle(any(AppAddedSchedulerEvent.class)); + verify(scheduler).handle(any(AppAttemptAddedSchedulerEvent.class)); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index a8041384079..c2f2f88dc46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -65,7 +65,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -555,8 +555,8 @@ public class TestCapacityScheduler { ApplicationId appId = BuilderUtils.newApplicationId(100, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); - SchedulerEvent event = new AppAddedSchedulerEvent(appAttemptId, "default", - "user"); + SchedulerEvent event = + new AppAttemptAddedSchedulerEvent(appAttemptId, "default", "user"); cs.handle(event); // Verify the blacklist can be updated independent of requesting containers diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 0ecb230f153..73eb697e0ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -63,7 +63,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -348,7 +348,7 @@ public class TestLeafQueue { a.submitApplication(app_0, user_0, B); when(cs.getApplication(appAttemptId_0)).thenReturn(app_0); - AppRemovedSchedulerEvent event = new AppRemovedSchedulerEvent( + AppAttemptRemovedSchedulerEvent event = new AppAttemptRemovedSchedulerEvent( appAttemptId_0, RMAppAttemptState.FAILED); cs.handle(event); @@ -366,7 +366,7 @@ public class TestLeafQueue { assertEquals(1, a.getMetrics().getAppsPending()); when(cs.getApplication(appAttemptId_1)).thenReturn(app_0); - event = new AppRemovedSchedulerEvent(appAttemptId_0, + event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0, RMAppAttemptState.FINISHED); cs.handle(event); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 84ffbc173af..b601b3883e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -19,10 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; import java.io.File; @@ -33,11 +33,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Set; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import javax.xml.parsers.ParserConfigurationException; @@ -54,10 +54,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; -import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -79,9 +79,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -256,7 +255,7 @@ public class TestFairScheduler { private ApplicationAttemptId createSchedulingRequest(int memory, int vcores, String queueId, String userId, int numContainers, int priority) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - scheduler.addApplication(id, queueId, userId); + scheduler.addApplicationAttempt(id, queueId, userId); List ask = new ArrayList(); ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, priority, numContainers, true); @@ -619,8 +618,8 @@ public class TestFairScheduler { null, null, null, false, false, 0, null, null), null, null, 0, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); - AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( - appAttemptId, "default", "user1"); + AppAttemptAddedSchedulerEvent appAddedEvent = + new AppAttemptAddedSchedulerEvent(appAttemptId, "default", "user1"); scheduler.handle(appAddedEvent); assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getRunnableAppSchedulables().size()); @@ -641,8 +640,8 @@ public class TestFairScheduler { null, null, null, false, false, 0, null, null), null, null, 0, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); - AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent( - appAttemptId, "default", "user2"); + AppAttemptAddedSchedulerEvent appAddedEvent2 = + new AppAttemptAddedSchedulerEvent(appAttemptId, "default", "user2"); scheduler.handle(appAddedEvent2); assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true) .getRunnableAppSchedulables().size()); @@ -661,8 +660,8 @@ public class TestFairScheduler { // submit app with empty queue ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); - AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( - appAttemptId, "", "user1"); + AppAttemptAddedSchedulerEvent appAddedEvent = + new AppAttemptAddedSchedulerEvent(appAttemptId, "", "user1"); scheduler.handle(appAddedEvent); // submission rejected @@ -787,11 +786,11 @@ public class TestFairScheduler { scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId id11 = createAppAttemptId(1, 1); - scheduler.addApplication(id11, "root.queue1", "user1"); + scheduler.addApplicationAttempt(id11, "root.queue1", "user1"); ApplicationAttemptId id21 = createAppAttemptId(2, 1); - scheduler.addApplication(id21, "root.queue2", "user1"); + scheduler.addApplicationAttempt(id21, "root.queue2", "user1"); ApplicationAttemptId id22 = createAppAttemptId(2, 2); - scheduler.addApplication(id22, "root.queue2", "user1"); + scheduler.addApplicationAttempt(id22, "root.queue2", "user1"); int minReqSize = FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB; @@ -833,8 +832,9 @@ public class TestFairScheduler { public void testAppAdditionAndRemoval() throws Exception { scheduler.reinitialize(conf, resourceManager.getRMContext()); - AppAddedSchedulerEvent appAddedEvent1 = new AppAddedSchedulerEvent( - createAppAttemptId(1, 1), "default", "user1"); + AppAttemptAddedSchedulerEvent appAddedEvent1 = + new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), "default", + "user1"); scheduler.handle(appAddedEvent1); // Scheduler should have two queues (the default and the one created for user1) @@ -844,7 +844,7 @@ public class TestFairScheduler { assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getRunnableAppSchedulables().size()); - AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent( createAppAttemptId(1, 1), RMAppAttemptState.FINISHED); // Now remove app @@ -1526,7 +1526,7 @@ public class TestFairScheduler { scheduler.handle(nodeEvent2); ApplicationAttemptId appId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - scheduler.addApplication(appId, "queue1", "user1"); + scheduler.addApplicationAttempt(appId, "queue1", "user1"); // 1 request with 2 nodes on the same rack. another request with 1 node on // a different rack @@ -1764,7 +1764,7 @@ public class TestFairScheduler { ApplicationAttemptId attId = ApplicationAttemptId.newInstance(applicationId, this.ATTEMPT_ID++); - scheduler.addApplication(attId, queue, user); + scheduler.addApplicationAttempt(attId, queue, user); numTries = 0; while (application.getFinishTime() == 0 && numTries < MAX_TRIES) { @@ -2238,8 +2238,8 @@ public class TestFairScheduler { verifyQueueNumRunnable("queue1", 2, 1); // Remove app 1 and both app 2 and app 4 should becomes runnable in its place - AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( - attId1, RMAppAttemptState.FINISHED); + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = + new AppAttemptRemovedSchedulerEvent(attId1, RMAppAttemptState.FINISHED); scheduler.handle(appRemovedEvent1); verifyAppRunnable(attId2, true); verifyQueueNumRunnable("queue2", 1, 0); @@ -2302,8 +2302,8 @@ public class TestFairScheduler { // Even though the app was removed from sub3, the app from sub2 gets to go // because it came in first - AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( - attId2, RMAppAttemptState.FINISHED); + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = + new AppAttemptRemovedSchedulerEvent(attId2, RMAppAttemptState.FINISHED); scheduler.handle(appRemovedEvent1); verifyAppRunnable(attId4, true); verifyQueueNumRunnable("queue1.sub2", 2, 0); @@ -2311,16 +2311,16 @@ public class TestFairScheduler { verifyQueueNumRunnable("queue1.sub3", 0, 1); // Now test removal of a non-runnable app - AppRemovedSchedulerEvent appRemovedEvent2 = new AppRemovedSchedulerEvent( - attId5, RMAppAttemptState.KILLED); + AppAttemptRemovedSchedulerEvent appRemovedEvent2 = + new AppAttemptRemovedSchedulerEvent(attId5, RMAppAttemptState.KILLED); scheduler.handle(appRemovedEvent2); assertEquals(0, scheduler.maxRunningEnforcer.usersNonRunnableApps .get("user1").size()); // verify app gone in queue accounting verifyQueueNumRunnable("queue1.sub3", 0, 0); // verify it doesn't become runnable when there would be space for it - AppRemovedSchedulerEvent appRemovedEvent3 = new AppRemovedSchedulerEvent( - attId4, RMAppAttemptState.FINISHED); + AppAttemptRemovedSchedulerEvent appRemovedEvent3 = + new AppAttemptRemovedSchedulerEvent(attId4, RMAppAttemptState.FINISHED); scheduler.handle(appRemovedEvent3); verifyQueueNumRunnable("queue1.sub2", 1, 0); verifyQueueNumRunnable("queue1.sub3", 0, 0); @@ -2356,7 +2356,7 @@ public class TestFairScheduler { // send application request ApplicationAttemptId appAttemptId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - fs.addApplication(appAttemptId, "queue11", "user11"); + fs.addApplicationAttempt(appAttemptId, "queue11", "user11"); List ask = new ArrayList(); ResourceRequest request = createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index e70070434ec..7ce7e42bc67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -64,7 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppRepor import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; @@ -150,13 +150,13 @@ public class TestFifoScheduler { ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); - SchedulerEvent event = new AppAddedSchedulerEvent(appAttemptId, "queue", - "user"); + SchedulerEvent event = + new AppAttemptAddedSchedulerEvent(appAttemptId, "queue", "user"); schedular.handle(event); appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2); - event = new AppAddedSchedulerEvent(appAttemptId, "queue", "user"); + event = new AppAttemptAddedSchedulerEvent(appAttemptId, "queue", "user"); schedular.handle(event); int afterAppsSubmitted = metrics.getAppsSubmitted(); @@ -188,8 +188,8 @@ public class TestFifoScheduler { int _appAttemptId = 1; ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId); - AppAddedSchedulerEvent appEvent1 = new AppAddedSchedulerEvent(appAttemptId, - "queue1", "user1"); + AppAttemptAddedSchedulerEvent appEvent1 = + new AppAttemptAddedSchedulerEvent(appAttemptId, "queue1", "user1"); scheduler.handle(appEvent1); int memory = 64; @@ -274,8 +274,8 @@ public class TestFifoScheduler { int _appAttemptId = 1; ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId); - AppAddedSchedulerEvent appEvent1 = new AppAddedSchedulerEvent(appAttemptId, - "queue1", "user1"); + AppAttemptAddedSchedulerEvent appEvent1 = + new AppAttemptAddedSchedulerEvent(appAttemptId, "queue1", "user1"); scheduler.handle(appEvent1); int memory = 1024; @@ -541,8 +541,8 @@ public class TestFifoScheduler { ApplicationId appId = BuilderUtils.newApplicationId(100, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); - SchedulerEvent event = new AppAddedSchedulerEvent(appAttemptId, "default", - "user"); + SchedulerEvent event = + new AppAttemptAddedSchedulerEvent(appAttemptId, "default", "user"); fs.handle(event); // Verify the blacklist can be updated independent of requesting containers From 791e43c53b0be23b03d9b3f7d45a0df615eb90da Mon Sep 17 00:00:00 2001 From: Jian He Date: Thu, 12 Dec 2013 23:45:01 +0000 Subject: [PATCH 73/76] Reverting YARN-1311. Fixed app specific scheduler-events' names to be app-attempt based. Contributed by Vinod Kumar Vavilapalli git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550594 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 - .../rmapp/attempt/RMAppAttemptImpl.java | 12 ++-- .../scheduler/capacity/CapacityScheduler.java | 26 ++++---- ...Event.java => AppAddedSchedulerEvent.java} | 8 +-- ...ent.java => AppRemovedSchedulerEvent.java} | 7 +-- .../scheduler/event/SchedulerEventType.java | 6 +- .../scheduler/fair/FairScheduler.java | 32 +++++----- .../scheduler/fifo/FifoScheduler.java | 41 +++++-------- .../server/resourcemanager/Application.java | 4 +- .../resourcemanager/TestFifoScheduler.java | 14 ++--- .../resourcemanager/TestResourceManager.java | 7 +-- .../attempt/TestRMAppAttemptTransitions.java | 4 +- .../capacity/TestCapacityScheduler.java | 6 +- .../scheduler/capacity/TestLeafQueue.java | 6 +- .../scheduler/fair/TestFairScheduler.java | 60 +++++++++---------- .../scheduler/fifo/TestFifoScheduler.java | 20 +++---- 16 files changed, 118 insertions(+), 138 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/{AppAttemptAddedSchedulerEvent.java => AppAddedSchedulerEvent.java} (85%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/{AppAttemptRemovedSchedulerEvent.java => AppRemovedSchedulerEvent.java} (87%) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 15a2d8572c6..ce954cf4cc4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -165,9 +165,6 @@ Release 2.4.0 - UNRELEASED YARN-1325. Modified RM HA configuration validation to also ensure that multiple RMs are configured. (Xuan Gong via vinodkv) - YARN-1311. Fixed app specific scheduler-events' names to be app-attempt - based. (vinodkv via jianhe) - OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 394465e984c..67a2e414583 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -87,8 +87,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; @@ -742,9 +742,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { new Token(id, appAttempt.rmContext.getAMRMTokenSecretManager()); - // Add the applicationAttempt to the scheduler + // Add the application to the scheduler appAttempt.eventHandler.handle( - new AppAttemptAddedSchedulerEvent(appAttempt.applicationAttemptId, + new AppAddedSchedulerEvent(appAttempt.applicationAttemptId, appAttempt.submissionContext.getQueue(), appAttempt.user)); } } @@ -1055,8 +1055,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } appAttempt.eventHandler.handle(appEvent); - appAttempt.eventHandler.handle(new AppAttemptRemovedSchedulerEvent( - appAttemptId, finalAttemptState)); + appAttempt.eventHandler.handle(new AppRemovedSchedulerEvent(appAttemptId, + finalAttemptState)); appAttempt.removeCredentials(appAttempt); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 60256398461..3a2b353363a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -71,8 +71,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeRepo import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; @@ -417,7 +417,7 @@ public class CapacityScheduler } private synchronized void - addApplicationAttempt(ApplicationAttemptId applicationAttemptId, + addApplication(ApplicationAttemptId applicationAttemptId, String queueName, String user) { // Sanity checks @@ -466,7 +466,7 @@ public class CapacityScheduler RMAppAttemptEventType.APP_ACCEPTED)); } - private synchronized void doneApplicationAttempt( + private synchronized void doneApplication( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState) { LOG.info("Application " + applicationAttemptId + " is done." + @@ -740,20 +740,18 @@ public class CapacityScheduler nodeUpdate(nodeUpdatedEvent.getRMNode()); } break; - case APP_ATTEMPT_ADDED: + case APP_ADDED: { - AppAttemptAddedSchedulerEvent appAttemptAddedEvent = - (AppAttemptAddedSchedulerEvent) event; - addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(), - appAttemptAddedEvent.getQueue(), appAttemptAddedEvent.getUser()); + AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent)event; + addApplication(appAddedEvent.getApplicationAttemptId(), appAddedEvent + .getQueue(), appAddedEvent.getUser()); } break; - case APP_ATTEMPT_REMOVED: + case APP_REMOVED: { - AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = - (AppAttemptRemovedSchedulerEvent) event; - doneApplicationAttempt(appAttemptRemovedEvent.getApplicationAttemptID(), - appAttemptRemovedEvent.getFinalAttemptState()); + AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event; + doneApplication(appRemovedEvent.getApplicationAttemptID(), + appRemovedEvent.getFinalAttemptState()); } break; case CONTAINER_EXPIRED: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptAddedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java similarity index 85% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptAddedSchedulerEvent.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java index 7b9ffff4b53..16211f31662 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptAddedSchedulerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java @@ -20,15 +20,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -public class AppAttemptAddedSchedulerEvent extends SchedulerEvent { +public class AppAddedSchedulerEvent extends SchedulerEvent { private final ApplicationAttemptId applicationAttemptId; private final String queue; private final String user; - public AppAttemptAddedSchedulerEvent( - ApplicationAttemptId applicationAttemptId, String queue, String user) { - super(SchedulerEventType.APP_ATTEMPT_ADDED); + public AppAddedSchedulerEvent(ApplicationAttemptId applicationAttemptId, + String queue, String user) { + super(SchedulerEventType.APP_ADDED); this.applicationAttemptId = applicationAttemptId; this.queue = queue; this.user = user; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptRemovedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppRemovedSchedulerEvent.java similarity index 87% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptRemovedSchedulerEvent.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppRemovedSchedulerEvent.java index 876d164b63a..963bc57b639 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptRemovedSchedulerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppRemovedSchedulerEvent.java @@ -21,15 +21,14 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; -public class AppAttemptRemovedSchedulerEvent extends SchedulerEvent { +public class AppRemovedSchedulerEvent extends SchedulerEvent { private final ApplicationAttemptId applicationAttemptId; private final RMAppAttemptState finalAttemptState; - public AppAttemptRemovedSchedulerEvent( - ApplicationAttemptId applicationAttemptId, + public AppRemovedSchedulerEvent(ApplicationAttemptId applicationAttemptId, RMAppAttemptState finalAttemptState) { - super(SchedulerEventType.APP_ATTEMPT_REMOVED); + super(SchedulerEventType.APP_REMOVED); this.applicationAttemptId = applicationAttemptId; this.finalAttemptState = finalAttemptState; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java index dd1aec71bff..a78918dc4e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java @@ -25,9 +25,9 @@ public enum SchedulerEventType { NODE_REMOVED, NODE_UPDATE, - // Source: RMAppAttempt - APP_ATTEMPT_ADDED, - APP_ATTEMPT_REMOVED, + // Source: App + APP_ADDED, + APP_REMOVED, // Source: ContainerAllocationExpirer CONTAINER_EXPIRED diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index bc716c1401f..861fad87158 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -75,8 +75,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppRepor import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; @@ -591,7 +591,7 @@ public class FairScheduler implements ResourceScheduler { * user. This will accept a new app even if the user or queue is above * configured limits, but the app will not be marked as runnable. */ - protected synchronized void addApplicationAttempt( + protected synchronized void addApplication( ApplicationAttemptId applicationAttemptId, String queueName, String user) { if (queueName == null || queueName.isEmpty()) { String message = "Reject application " + applicationAttemptId + @@ -674,7 +674,7 @@ public class FairScheduler implements ResourceScheduler { return queue; } - private synchronized void removeApplicationAttempt( + private synchronized void removeApplication( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState) { LOG.info("Application " + applicationAttemptId + " is done." + @@ -1090,24 +1090,22 @@ public class FairScheduler implements ResourceScheduler { NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event; nodeUpdate(nodeUpdatedEvent.getRMNode()); break; - case APP_ATTEMPT_ADDED: - if (!(event instanceof AppAttemptAddedSchedulerEvent)) { + case APP_ADDED: + if (!(event instanceof AppAddedSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); } - AppAttemptAddedSchedulerEvent appAttemptAddedEvent = - (AppAttemptAddedSchedulerEvent) event; - String queue = appAttemptAddedEvent.getQueue(); - addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(), - queue, appAttemptAddedEvent.getUser()); + AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent)event; + String queue = appAddedEvent.getQueue(); + addApplication(appAddedEvent.getApplicationAttemptId(), queue, + appAddedEvent.getUser()); break; - case APP_ATTEMPT_REMOVED: - if (!(event instanceof AppAttemptRemovedSchedulerEvent)) { + case APP_REMOVED: + if (!(event instanceof AppRemovedSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); } - AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = - (AppAttemptRemovedSchedulerEvent) event; - removeApplicationAttempt(appAttemptRemovedEvent.getApplicationAttemptID(), - appAttemptRemovedEvent.getFinalAttemptState()); + AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event; + removeApplication(appRemovedEvent.getApplicationAttemptID(), + appRemovedEvent.getFinalAttemptState()); break; case CONTAINER_EXPIRED: if (!(event instanceof ContainerExpiredSchedulerEvent)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index c5f0bd67d92..956cb492f56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -66,20 +67,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; @@ -348,7 +340,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { return nodes.get(nodeId); } - private synchronized void addApplicationAttempt(ApplicationAttemptId appAttemptId, + private synchronized void addApplication(ApplicationAttemptId appAttemptId, String user) { // TODO: Fix store FiCaSchedulerApp schedulerApp = @@ -363,7 +355,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { RMAppAttemptEventType.APP_ACCEPTED)); } - private synchronized void doneApplicationAttempt( + private synchronized void doneApplication( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState) throws IOException { @@ -697,25 +689,22 @@ public class FifoScheduler implements ResourceScheduler, Configurable { nodeUpdate(nodeUpdatedEvent.getRMNode()); } break; - case APP_ATTEMPT_ADDED: + case APP_ADDED: { - AppAttemptAddedSchedulerEvent appAttemptAddedEvent = - (AppAttemptAddedSchedulerEvent) event; - addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(), - appAttemptAddedEvent.getUser()); + AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event; + addApplication(appAddedEvent.getApplicationAttemptId(), appAddedEvent + .getUser()); } break; - case APP_ATTEMPT_REMOVED: + case APP_REMOVED: { - AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = - (AppAttemptRemovedSchedulerEvent) event; + AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event; try { - doneApplicationAttempt( - appAttemptRemovedEvent.getApplicationAttemptID(), - appAttemptRemovedEvent.getFinalAttemptState()); + doneApplication(appRemovedEvent.getApplicationAttemptID(), + appRemovedEvent.getFinalAttemptState()); } catch(IOException ie) { LOG.error("Unable to remove application " - + appAttemptRemovedEvent.getApplicationAttemptID(), ie); + + appRemovedEvent.getApplicationAttemptID(), ie); } } break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index 6767180b62a..cbce6e30d58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -57,7 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.Task.State; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; @@ -166,7 +166,7 @@ public class Application { resourceManager.getClientRMService().submitApplication(request); // Notify scheduler - AppAttemptAddedSchedulerEvent appAddedEvent1 = new AppAttemptAddedSchedulerEvent( + AppAddedSchedulerEvent appAddedEvent1 = new AppAddedSchedulerEvent( this.applicationAttemptId, this.queue, this.user); scheduler.handle(appAddedEvent1); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index 2d370fcfb46..c57a7529a3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -297,8 +297,8 @@ public class TestFifoScheduler { ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1); ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId( appId1, 1); - SchedulerEvent event1 = - new AppAttemptAddedSchedulerEvent(appAttemptId1, "queue", "user"); + SchedulerEvent event1 = new AppAddedSchedulerEvent(appAttemptId1, "queue", + "user"); fs.handle(event1); List emptyId = new ArrayList(); @@ -388,15 +388,15 @@ public class TestFifoScheduler { ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1); ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId( appId1, 1); - SchedulerEvent event1 = - new AppAttemptAddedSchedulerEvent(appAttemptId1, "queue", "user"); + SchedulerEvent event1 = new AppAddedSchedulerEvent(appAttemptId1, "queue", + "user"); fs.handle(event1); ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2); ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId( appId2, 1); - SchedulerEvent event2 = - new AppAttemptAddedSchedulerEvent(appAttemptId2, "queue", "user"); + SchedulerEvent event2 = new AppAddedSchedulerEvent(appAttemptId2, "queue", + "user"); fs.handle(event2); List emptyId = new ArrayList(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index 3bcfba3abd5..3ed92b69664 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.util.resource.Resources; @@ -159,9 +159,8 @@ public class TestResourceManager { application.finishTask(t3); // Notify scheduler application is finished. - AppAttemptRemovedSchedulerEvent appRemovedEvent1 = - new AppAttemptRemovedSchedulerEvent( - application.getApplicationAttemptId(), RMAppAttemptState.FINISHED); + AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( + application.getApplicationAttemptId(), RMAppAttemptState.FINISHED); resourceManager.getResourceScheduler().handle(appRemovedEvent1); checkResourceUsage(nm1, nm2); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 6c72d97080d..140b53e62ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -85,7 +85,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; @@ -329,7 +329,7 @@ public class TestRMAppAttemptTransitions { // Check events verify(masterService). registerAppAttempt(applicationAttempt.getAppAttemptId()); - verify(scheduler).handle(any(AppAttemptAddedSchedulerEvent.class)); + verify(scheduler).handle(any(AppAddedSchedulerEvent.class)); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index c2f2f88dc46..a8041384079 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -65,7 +65,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -555,8 +555,8 @@ public class TestCapacityScheduler { ApplicationId appId = BuilderUtils.newApplicationId(100, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); - SchedulerEvent event = - new AppAttemptAddedSchedulerEvent(appAttemptId, "default", "user"); + SchedulerEvent event = new AppAddedSchedulerEvent(appAttemptId, "default", + "user"); cs.handle(event); // Verify the blacklist can be updated independent of requesting containers diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 73eb697e0ec..0ecb230f153 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -63,7 +63,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -348,7 +348,7 @@ public class TestLeafQueue { a.submitApplication(app_0, user_0, B); when(cs.getApplication(appAttemptId_0)).thenReturn(app_0); - AppAttemptRemovedSchedulerEvent event = new AppAttemptRemovedSchedulerEvent( + AppRemovedSchedulerEvent event = new AppRemovedSchedulerEvent( appAttemptId_0, RMAppAttemptState.FAILED); cs.handle(event); @@ -366,7 +366,7 @@ public class TestLeafQueue { assertEquals(1, a.getMetrics().getAppsPending()); when(cs.getApplication(appAttemptId_1)).thenReturn(app_0); - event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0, + event = new AppRemovedSchedulerEvent(appAttemptId_0, RMAppAttemptState.FINISHED); cs.handle(event); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index b601b3883e9..84ffbc173af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -19,10 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; import java.io.File; @@ -33,11 +33,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Set; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import javax.xml.parsers.ParserConfigurationException; @@ -54,10 +54,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -79,8 +79,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -255,7 +256,7 @@ public class TestFairScheduler { private ApplicationAttemptId createSchedulingRequest(int memory, int vcores, String queueId, String userId, int numContainers, int priority) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - scheduler.addApplicationAttempt(id, queueId, userId); + scheduler.addApplication(id, queueId, userId); List ask = new ArrayList(); ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, priority, numContainers, true); @@ -618,8 +619,8 @@ public class TestFairScheduler { null, null, null, false, false, 0, null, null), null, null, 0, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); - AppAttemptAddedSchedulerEvent appAddedEvent = - new AppAttemptAddedSchedulerEvent(appAttemptId, "default", "user1"); + AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( + appAttemptId, "default", "user1"); scheduler.handle(appAddedEvent); assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getRunnableAppSchedulables().size()); @@ -640,8 +641,8 @@ public class TestFairScheduler { null, null, null, false, false, 0, null, null), null, null, 0, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); - AppAttemptAddedSchedulerEvent appAddedEvent2 = - new AppAttemptAddedSchedulerEvent(appAttemptId, "default", "user2"); + AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent( + appAttemptId, "default", "user2"); scheduler.handle(appAddedEvent2); assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true) .getRunnableAppSchedulables().size()); @@ -660,8 +661,8 @@ public class TestFairScheduler { // submit app with empty queue ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); - AppAttemptAddedSchedulerEvent appAddedEvent = - new AppAttemptAddedSchedulerEvent(appAttemptId, "", "user1"); + AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( + appAttemptId, "", "user1"); scheduler.handle(appAddedEvent); // submission rejected @@ -786,11 +787,11 @@ public class TestFairScheduler { scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId id11 = createAppAttemptId(1, 1); - scheduler.addApplicationAttempt(id11, "root.queue1", "user1"); + scheduler.addApplication(id11, "root.queue1", "user1"); ApplicationAttemptId id21 = createAppAttemptId(2, 1); - scheduler.addApplicationAttempt(id21, "root.queue2", "user1"); + scheduler.addApplication(id21, "root.queue2", "user1"); ApplicationAttemptId id22 = createAppAttemptId(2, 2); - scheduler.addApplicationAttempt(id22, "root.queue2", "user1"); + scheduler.addApplication(id22, "root.queue2", "user1"); int minReqSize = FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB; @@ -832,9 +833,8 @@ public class TestFairScheduler { public void testAppAdditionAndRemoval() throws Exception { scheduler.reinitialize(conf, resourceManager.getRMContext()); - AppAttemptAddedSchedulerEvent appAddedEvent1 = - new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), "default", - "user1"); + AppAddedSchedulerEvent appAddedEvent1 = new AppAddedSchedulerEvent( + createAppAttemptId(1, 1), "default", "user1"); scheduler.handle(appAddedEvent1); // Scheduler should have two queues (the default and the one created for user1) @@ -844,7 +844,7 @@ public class TestFairScheduler { assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getRunnableAppSchedulables().size()); - AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent( + AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( createAppAttemptId(1, 1), RMAppAttemptState.FINISHED); // Now remove app @@ -1526,7 +1526,7 @@ public class TestFairScheduler { scheduler.handle(nodeEvent2); ApplicationAttemptId appId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - scheduler.addApplicationAttempt(appId, "queue1", "user1"); + scheduler.addApplication(appId, "queue1", "user1"); // 1 request with 2 nodes on the same rack. another request with 1 node on // a different rack @@ -1764,7 +1764,7 @@ public class TestFairScheduler { ApplicationAttemptId attId = ApplicationAttemptId.newInstance(applicationId, this.ATTEMPT_ID++); - scheduler.addApplicationAttempt(attId, queue, user); + scheduler.addApplication(attId, queue, user); numTries = 0; while (application.getFinishTime() == 0 && numTries < MAX_TRIES) { @@ -2238,8 +2238,8 @@ public class TestFairScheduler { verifyQueueNumRunnable("queue1", 2, 1); // Remove app 1 and both app 2 and app 4 should becomes runnable in its place - AppAttemptRemovedSchedulerEvent appRemovedEvent1 = - new AppAttemptRemovedSchedulerEvent(attId1, RMAppAttemptState.FINISHED); + AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( + attId1, RMAppAttemptState.FINISHED); scheduler.handle(appRemovedEvent1); verifyAppRunnable(attId2, true); verifyQueueNumRunnable("queue2", 1, 0); @@ -2302,8 +2302,8 @@ public class TestFairScheduler { // Even though the app was removed from sub3, the app from sub2 gets to go // because it came in first - AppAttemptRemovedSchedulerEvent appRemovedEvent1 = - new AppAttemptRemovedSchedulerEvent(attId2, RMAppAttemptState.FINISHED); + AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( + attId2, RMAppAttemptState.FINISHED); scheduler.handle(appRemovedEvent1); verifyAppRunnable(attId4, true); verifyQueueNumRunnable("queue1.sub2", 2, 0); @@ -2311,16 +2311,16 @@ public class TestFairScheduler { verifyQueueNumRunnable("queue1.sub3", 0, 1); // Now test removal of a non-runnable app - AppAttemptRemovedSchedulerEvent appRemovedEvent2 = - new AppAttemptRemovedSchedulerEvent(attId5, RMAppAttemptState.KILLED); + AppRemovedSchedulerEvent appRemovedEvent2 = new AppRemovedSchedulerEvent( + attId5, RMAppAttemptState.KILLED); scheduler.handle(appRemovedEvent2); assertEquals(0, scheduler.maxRunningEnforcer.usersNonRunnableApps .get("user1").size()); // verify app gone in queue accounting verifyQueueNumRunnable("queue1.sub3", 0, 0); // verify it doesn't become runnable when there would be space for it - AppAttemptRemovedSchedulerEvent appRemovedEvent3 = - new AppAttemptRemovedSchedulerEvent(attId4, RMAppAttemptState.FINISHED); + AppRemovedSchedulerEvent appRemovedEvent3 = new AppRemovedSchedulerEvent( + attId4, RMAppAttemptState.FINISHED); scheduler.handle(appRemovedEvent3); verifyQueueNumRunnable("queue1.sub2", 1, 0); verifyQueueNumRunnable("queue1.sub3", 0, 0); @@ -2356,7 +2356,7 @@ public class TestFairScheduler { // send application request ApplicationAttemptId appAttemptId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - fs.addApplicationAttempt(appAttemptId, "queue11", "user11"); + fs.addApplication(appAttemptId, "queue11", "user11"); List ask = new ArrayList(); ResourceRequest request = createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 7ce7e42bc67..e70070434ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -64,7 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppRepor import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; @@ -150,13 +150,13 @@ public class TestFifoScheduler { ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); - SchedulerEvent event = - new AppAttemptAddedSchedulerEvent(appAttemptId, "queue", "user"); + SchedulerEvent event = new AppAddedSchedulerEvent(appAttemptId, "queue", + "user"); schedular.handle(event); appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2); - event = new AppAttemptAddedSchedulerEvent(appAttemptId, "queue", "user"); + event = new AppAddedSchedulerEvent(appAttemptId, "queue", "user"); schedular.handle(event); int afterAppsSubmitted = metrics.getAppsSubmitted(); @@ -188,8 +188,8 @@ public class TestFifoScheduler { int _appAttemptId = 1; ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId); - AppAttemptAddedSchedulerEvent appEvent1 = - new AppAttemptAddedSchedulerEvent(appAttemptId, "queue1", "user1"); + AppAddedSchedulerEvent appEvent1 = new AppAddedSchedulerEvent(appAttemptId, + "queue1", "user1"); scheduler.handle(appEvent1); int memory = 64; @@ -274,8 +274,8 @@ public class TestFifoScheduler { int _appAttemptId = 1; ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId); - AppAttemptAddedSchedulerEvent appEvent1 = - new AppAttemptAddedSchedulerEvent(appAttemptId, "queue1", "user1"); + AppAddedSchedulerEvent appEvent1 = new AppAddedSchedulerEvent(appAttemptId, + "queue1", "user1"); scheduler.handle(appEvent1); int memory = 1024; @@ -541,8 +541,8 @@ public class TestFifoScheduler { ApplicationId appId = BuilderUtils.newApplicationId(100, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); - SchedulerEvent event = - new AppAttemptAddedSchedulerEvent(appAttemptId, "default", "user"); + SchedulerEvent event = new AppAddedSchedulerEvent(appAttemptId, "default", + "user"); fs.handle(event); // Verify the blacklist can be updated independent of requesting containers From 28db28d491b4e3e3ddf440db4cebab8c0ad6b7f3 Mon Sep 17 00:00:00 2001 From: Jian He Date: Fri, 13 Dec 2013 01:44:16 +0000 Subject: [PATCH 74/76] YARN-1311. Fixed app specific scheduler-events' names to be app-attempt based. Contributed by Vinod Kumar Vavilapalli git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550613 13f79535-47bb-0310-9956-ffa450edef68 --- .../scheduler/ResourceSchedulerWrapper.java | 134 ++++++++---------- .../rmapp/attempt/RMAppAttemptImpl.java | 12 +- .../scheduler/capacity/CapacityScheduler.java | 26 ++-- ...ava => AppAttemptAddedSchedulerEvent.java} | 8 +- ...a => AppAttemptRemovedSchedulerEvent.java} | 7 +- .../scheduler/event/SchedulerEventType.java | 6 +- .../scheduler/fair/FairScheduler.java | 32 +++-- .../scheduler/fifo/FifoScheduler.java | 41 ++++-- .../server/resourcemanager/Application.java | 4 +- .../resourcemanager/TestFifoScheduler.java | 14 +- .../resourcemanager/TestResourceManager.java | 7 +- .../attempt/TestRMAppAttemptTransitions.java | 4 +- .../capacity/TestCapacityScheduler.java | 6 +- .../scheduler/capacity/TestLeafQueue.java | 6 +- .../scheduler/fair/TestFairScheduler.java | 60 ++++---- .../scheduler/fifo/TestFifoScheduler.java | 20 +-- 16 files changed, 196 insertions(+), 191 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/{AppAddedSchedulerEvent.java => AppAttemptAddedSchedulerEvent.java} (85%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/{AppRemovedSchedulerEvent.java => AppAttemptRemovedSchedulerEvent.java} (87%) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index b7fdb8bfa45..ad066e75d28 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -17,67 +17,6 @@ */ package org.apache.hadoop.yarn.sls.scheduler; -import org.apache.hadoop.util.ShutdownHookManager; -import org.apache.hadoop.yarn.sls.SLSRunner; -import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; -import org.apache.hadoop.yarn.sls.web.SLSWebApp; -import com.codahale.metrics.Counter; -import com.codahale.metrics.CsvReporter; -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.SlidingWindowReservoir; -import com.codahale.metrics.Timer; - -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerExitStatus; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.QueueInfo; -import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; -import org.apache.hadoop.yarn.api.records.QueueACL; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode - .UpdatedContainerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler - .ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler - .SchedulerAppReport; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler - .SchedulerNodeReport; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity - .CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event - .AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event - .AppRemovedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event - .NodeUpdateSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event - .SchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event - .SchedulerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .FairScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo - .FifoScheduler; -import org.apache.hadoop.yarn.util.resource.Resources; -import org.apache.log4j.Logger; - import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; @@ -97,6 +36,55 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.ShutdownHookManager; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.sls.SLSRunner; +import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; +import org.apache.hadoop.yarn.sls.web.SLSWebApp; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.apache.log4j.Logger; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.CsvReporter; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.SlidingWindowReservoir; +import com.codahale.metrics.Timer; + public class ResourceSchedulerWrapper implements ResourceScheduler, Configurable { private static final String EOL = System.getProperty("line.separator"); @@ -246,11 +234,11 @@ public class ResourceSchedulerWrapper implements ResourceScheduler, (NodeUpdateSchedulerEvent)schedulerEvent); schedulerEvent = eventWrapper; updateQueueWithNodeUpdate(eventWrapper); - } else if (schedulerEvent.getType() == SchedulerEventType.APP_REMOVED - && schedulerEvent instanceof AppRemovedSchedulerEvent) { + } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED + && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) { // check if having AM Container, update resource usage information - AppRemovedSchedulerEvent appRemoveEvent = - (AppRemovedSchedulerEvent) schedulerEvent; + AppAttemptRemovedSchedulerEvent appRemoveEvent = + (AppAttemptRemovedSchedulerEvent) schedulerEvent; ApplicationAttemptId appAttemptId = appRemoveEvent.getApplicationAttemptID(); String queue = appQueueMap.get(appAttemptId); @@ -275,18 +263,18 @@ public class ResourceSchedulerWrapper implements ResourceScheduler, schedulerHandleCounter.inc(); schedulerHandleCounterMap.get(schedulerEvent.getType()).inc(); - if (schedulerEvent.getType() == SchedulerEventType.APP_REMOVED - && schedulerEvent instanceof AppRemovedSchedulerEvent) { + if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED + && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) { SLSRunner.decreaseRemainingApps(); - AppRemovedSchedulerEvent appRemoveEvent = - (AppRemovedSchedulerEvent) schedulerEvent; + AppAttemptRemovedSchedulerEvent appRemoveEvent = + (AppAttemptRemovedSchedulerEvent) schedulerEvent; ApplicationAttemptId appAttemptId = appRemoveEvent.getApplicationAttemptID(); appQueueMap.remove(appRemoveEvent.getApplicationAttemptID()); - } else if (schedulerEvent.getType() == SchedulerEventType.APP_ADDED - && schedulerEvent instanceof AppAddedSchedulerEvent) { - AppAddedSchedulerEvent appAddEvent = - (AppAddedSchedulerEvent) schedulerEvent; + } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_ADDED + && schedulerEvent instanceof AppAttemptAddedSchedulerEvent) { + AppAttemptAddedSchedulerEvent appAddEvent = + (AppAttemptAddedSchedulerEvent) schedulerEvent; String queueName = appAddEvent.getQueue(); appQueueMap.put(appAddEvent.getApplicationAttemptId(), queueName); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 67a2e414583..394465e984c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -87,8 +87,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; @@ -742,9 +742,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { new Token(id, appAttempt.rmContext.getAMRMTokenSecretManager()); - // Add the application to the scheduler + // Add the applicationAttempt to the scheduler appAttempt.eventHandler.handle( - new AppAddedSchedulerEvent(appAttempt.applicationAttemptId, + new AppAttemptAddedSchedulerEvent(appAttempt.applicationAttemptId, appAttempt.submissionContext.getQueue(), appAttempt.user)); } } @@ -1055,8 +1055,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } appAttempt.eventHandler.handle(appEvent); - appAttempt.eventHandler.handle(new AppRemovedSchedulerEvent(appAttemptId, - finalAttemptState)); + appAttempt.eventHandler.handle(new AppAttemptRemovedSchedulerEvent( + appAttemptId, finalAttemptState)); appAttempt.removeCredentials(appAttempt); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 3a2b353363a..60256398461 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -71,8 +71,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeRepo import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; @@ -417,7 +417,7 @@ public class CapacityScheduler } private synchronized void - addApplication(ApplicationAttemptId applicationAttemptId, + addApplicationAttempt(ApplicationAttemptId applicationAttemptId, String queueName, String user) { // Sanity checks @@ -466,7 +466,7 @@ public class CapacityScheduler RMAppAttemptEventType.APP_ACCEPTED)); } - private synchronized void doneApplication( + private synchronized void doneApplicationAttempt( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState) { LOG.info("Application " + applicationAttemptId + " is done." + @@ -740,18 +740,20 @@ public class CapacityScheduler nodeUpdate(nodeUpdatedEvent.getRMNode()); } break; - case APP_ADDED: + case APP_ATTEMPT_ADDED: { - AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent)event; - addApplication(appAddedEvent.getApplicationAttemptId(), appAddedEvent - .getQueue(), appAddedEvent.getUser()); + AppAttemptAddedSchedulerEvent appAttemptAddedEvent = + (AppAttemptAddedSchedulerEvent) event; + addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(), + appAttemptAddedEvent.getQueue(), appAttemptAddedEvent.getUser()); } break; - case APP_REMOVED: + case APP_ATTEMPT_REMOVED: { - AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event; - doneApplication(appRemovedEvent.getApplicationAttemptID(), - appRemovedEvent.getFinalAttemptState()); + AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = + (AppAttemptRemovedSchedulerEvent) event; + doneApplicationAttempt(appAttemptRemovedEvent.getApplicationAttemptID(), + appAttemptRemovedEvent.getFinalAttemptState()); } break; case CONTAINER_EXPIRED: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptAddedSchedulerEvent.java similarity index 85% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptAddedSchedulerEvent.java index 16211f31662..7b9ffff4b53 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptAddedSchedulerEvent.java @@ -20,15 +20,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -public class AppAddedSchedulerEvent extends SchedulerEvent { +public class AppAttemptAddedSchedulerEvent extends SchedulerEvent { private final ApplicationAttemptId applicationAttemptId; private final String queue; private final String user; - public AppAddedSchedulerEvent(ApplicationAttemptId applicationAttemptId, - String queue, String user) { - super(SchedulerEventType.APP_ADDED); + public AppAttemptAddedSchedulerEvent( + ApplicationAttemptId applicationAttemptId, String queue, String user) { + super(SchedulerEventType.APP_ATTEMPT_ADDED); this.applicationAttemptId = applicationAttemptId; this.queue = queue; this.user = user; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppRemovedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptRemovedSchedulerEvent.java similarity index 87% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppRemovedSchedulerEvent.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptRemovedSchedulerEvent.java index 963bc57b639..876d164b63a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppRemovedSchedulerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptRemovedSchedulerEvent.java @@ -21,14 +21,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; -public class AppRemovedSchedulerEvent extends SchedulerEvent { +public class AppAttemptRemovedSchedulerEvent extends SchedulerEvent { private final ApplicationAttemptId applicationAttemptId; private final RMAppAttemptState finalAttemptState; - public AppRemovedSchedulerEvent(ApplicationAttemptId applicationAttemptId, + public AppAttemptRemovedSchedulerEvent( + ApplicationAttemptId applicationAttemptId, RMAppAttemptState finalAttemptState) { - super(SchedulerEventType.APP_REMOVED); + super(SchedulerEventType.APP_ATTEMPT_REMOVED); this.applicationAttemptId = applicationAttemptId; this.finalAttemptState = finalAttemptState; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java index a78918dc4e9..dd1aec71bff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java @@ -25,9 +25,9 @@ public enum SchedulerEventType { NODE_REMOVED, NODE_UPDATE, - // Source: App - APP_ADDED, - APP_REMOVED, + // Source: RMAppAttempt + APP_ATTEMPT_ADDED, + APP_ATTEMPT_REMOVED, // Source: ContainerAllocationExpirer CONTAINER_EXPIRED diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 861fad87158..bc716c1401f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -75,8 +75,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppRepor import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; @@ -591,7 +591,7 @@ public class FairScheduler implements ResourceScheduler { * user. This will accept a new app even if the user or queue is above * configured limits, but the app will not be marked as runnable. */ - protected synchronized void addApplication( + protected synchronized void addApplicationAttempt( ApplicationAttemptId applicationAttemptId, String queueName, String user) { if (queueName == null || queueName.isEmpty()) { String message = "Reject application " + applicationAttemptId + @@ -674,7 +674,7 @@ public class FairScheduler implements ResourceScheduler { return queue; } - private synchronized void removeApplication( + private synchronized void removeApplicationAttempt( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState) { LOG.info("Application " + applicationAttemptId + " is done." + @@ -1090,22 +1090,24 @@ public class FairScheduler implements ResourceScheduler { NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event; nodeUpdate(nodeUpdatedEvent.getRMNode()); break; - case APP_ADDED: - if (!(event instanceof AppAddedSchedulerEvent)) { + case APP_ATTEMPT_ADDED: + if (!(event instanceof AppAttemptAddedSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); } - AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent)event; - String queue = appAddedEvent.getQueue(); - addApplication(appAddedEvent.getApplicationAttemptId(), queue, - appAddedEvent.getUser()); + AppAttemptAddedSchedulerEvent appAttemptAddedEvent = + (AppAttemptAddedSchedulerEvent) event; + String queue = appAttemptAddedEvent.getQueue(); + addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(), + queue, appAttemptAddedEvent.getUser()); break; - case APP_REMOVED: - if (!(event instanceof AppRemovedSchedulerEvent)) { + case APP_ATTEMPT_REMOVED: + if (!(event instanceof AppAttemptRemovedSchedulerEvent)) { throw new RuntimeException("Unexpected event type: " + event); } - AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event; - removeApplication(appRemovedEvent.getApplicationAttemptID(), - appRemovedEvent.getFinalAttemptState()); + AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = + (AppAttemptRemovedSchedulerEvent) event; + removeApplicationAttempt(appAttemptRemovedEvent.getApplicationAttemptID(), + appAttemptRemovedEvent.getFinalAttemptState()); break; case CONTAINER_EXPIRED: if (!(event instanceof ContainerExpiredSchedulerEvent)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 956cb492f56..c5f0bd67d92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -67,11 +66,20 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; @@ -340,7 +348,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { return nodes.get(nodeId); } - private synchronized void addApplication(ApplicationAttemptId appAttemptId, + private synchronized void addApplicationAttempt(ApplicationAttemptId appAttemptId, String user) { // TODO: Fix store FiCaSchedulerApp schedulerApp = @@ -355,7 +363,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { RMAppAttemptEventType.APP_ACCEPTED)); } - private synchronized void doneApplication( + private synchronized void doneApplicationAttempt( ApplicationAttemptId applicationAttemptId, RMAppAttemptState rmAppAttemptFinalState) throws IOException { @@ -689,22 +697,25 @@ public class FifoScheduler implements ResourceScheduler, Configurable { nodeUpdate(nodeUpdatedEvent.getRMNode()); } break; - case APP_ADDED: + case APP_ATTEMPT_ADDED: { - AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event; - addApplication(appAddedEvent.getApplicationAttemptId(), appAddedEvent - .getUser()); + AppAttemptAddedSchedulerEvent appAttemptAddedEvent = + (AppAttemptAddedSchedulerEvent) event; + addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(), + appAttemptAddedEvent.getUser()); } break; - case APP_REMOVED: + case APP_ATTEMPT_REMOVED: { - AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event; + AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = + (AppAttemptRemovedSchedulerEvent) event; try { - doneApplication(appRemovedEvent.getApplicationAttemptID(), - appRemovedEvent.getFinalAttemptState()); + doneApplicationAttempt( + appAttemptRemovedEvent.getApplicationAttemptID(), + appAttemptRemovedEvent.getFinalAttemptState()); } catch(IOException ie) { LOG.error("Unable to remove application " - + appRemovedEvent.getApplicationAttemptID(), ie); + + appAttemptRemovedEvent.getApplicationAttemptID(), ie); } } break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index cbce6e30d58..6767180b62a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -57,7 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.Task.State; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; @@ -166,7 +166,7 @@ public class Application { resourceManager.getClientRMService().submitApplication(request); // Notify scheduler - AppAddedSchedulerEvent appAddedEvent1 = new AppAddedSchedulerEvent( + AppAttemptAddedSchedulerEvent appAddedEvent1 = new AppAttemptAddedSchedulerEvent( this.applicationAttemptId, this.queue, this.user); scheduler.handle(appAddedEvent1); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index c57a7529a3f..2d370fcfb46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -297,8 +297,8 @@ public class TestFifoScheduler { ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1); ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId( appId1, 1); - SchedulerEvent event1 = new AppAddedSchedulerEvent(appAttemptId1, "queue", - "user"); + SchedulerEvent event1 = + new AppAttemptAddedSchedulerEvent(appAttemptId1, "queue", "user"); fs.handle(event1); List emptyId = new ArrayList(); @@ -388,15 +388,15 @@ public class TestFifoScheduler { ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1); ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId( appId1, 1); - SchedulerEvent event1 = new AppAddedSchedulerEvent(appAttemptId1, "queue", - "user"); + SchedulerEvent event1 = + new AppAttemptAddedSchedulerEvent(appAttemptId1, "queue", "user"); fs.handle(event1); ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2); ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId( appId2, 1); - SchedulerEvent event2 = new AppAddedSchedulerEvent(appAttemptId2, "queue", - "user"); + SchedulerEvent event2 = + new AppAttemptAddedSchedulerEvent(appAttemptId2, "queue", "user"); fs.handle(event2); List emptyId = new ArrayList(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index 3ed92b69664..3bcfba3abd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.util.resource.Resources; @@ -159,8 +159,9 @@ public class TestResourceManager { application.finishTask(t3); // Notify scheduler application is finished. - AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( - application.getApplicationAttemptId(), RMAppAttemptState.FINISHED); + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = + new AppAttemptRemovedSchedulerEvent( + application.getApplicationAttemptId(), RMAppAttemptState.FINISHED); resourceManager.getResourceScheduler().handle(appRemovedEvent1); checkResourceUsage(nm1, nm2); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 140b53e62ff..6c72d97080d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -85,7 +85,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; @@ -329,7 +329,7 @@ public class TestRMAppAttemptTransitions { // Check events verify(masterService). registerAppAttempt(applicationAttempt.getAppAttemptId()); - verify(scheduler).handle(any(AppAddedSchedulerEvent.class)); + verify(scheduler).handle(any(AppAttemptAddedSchedulerEvent.class)); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index a8041384079..c2f2f88dc46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -65,7 +65,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -555,8 +555,8 @@ public class TestCapacityScheduler { ApplicationId appId = BuilderUtils.newApplicationId(100, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); - SchedulerEvent event = new AppAddedSchedulerEvent(appAttemptId, "default", - "user"); + SchedulerEvent event = + new AppAttemptAddedSchedulerEvent(appAttemptId, "default", "user"); cs.handle(event); // Verify the blacklist can be updated independent of requesting containers diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 0ecb230f153..73eb697e0ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -63,7 +63,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; @@ -348,7 +348,7 @@ public class TestLeafQueue { a.submitApplication(app_0, user_0, B); when(cs.getApplication(appAttemptId_0)).thenReturn(app_0); - AppRemovedSchedulerEvent event = new AppRemovedSchedulerEvent( + AppAttemptRemovedSchedulerEvent event = new AppAttemptRemovedSchedulerEvent( appAttemptId_0, RMAppAttemptState.FAILED); cs.handle(event); @@ -366,7 +366,7 @@ public class TestLeafQueue { assertEquals(1, a.getMetrics().getAppsPending()); when(cs.getApplication(appAttemptId_1)).thenReturn(app_0); - event = new AppRemovedSchedulerEvent(appAttemptId_0, + event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0, RMAppAttemptState.FINISHED); cs.handle(event); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 84ffbc173af..b601b3883e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -19,10 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; import java.io.File; @@ -33,11 +33,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Set; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import javax.xml.parsers.ParserConfigurationException; @@ -54,10 +54,10 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl; -import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -79,9 +79,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -256,7 +255,7 @@ public class TestFairScheduler { private ApplicationAttemptId createSchedulingRequest(int memory, int vcores, String queueId, String userId, int numContainers, int priority) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - scheduler.addApplication(id, queueId, userId); + scheduler.addApplicationAttempt(id, queueId, userId); List ask = new ArrayList(); ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, priority, numContainers, true); @@ -619,8 +618,8 @@ public class TestFairScheduler { null, null, null, false, false, 0, null, null), null, null, 0, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); - AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( - appAttemptId, "default", "user1"); + AppAttemptAddedSchedulerEvent appAddedEvent = + new AppAttemptAddedSchedulerEvent(appAttemptId, "default", "user1"); scheduler.handle(appAddedEvent); assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getRunnableAppSchedulables().size()); @@ -641,8 +640,8 @@ public class TestFairScheduler { null, null, null, false, false, 0, null, null), null, null, 0, null); appsMap.put(appAttemptId.getApplicationId(), rmApp); - AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent( - appAttemptId, "default", "user2"); + AppAttemptAddedSchedulerEvent appAddedEvent2 = + new AppAttemptAddedSchedulerEvent(appAttemptId, "default", "user2"); scheduler.handle(appAddedEvent2); assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true) .getRunnableAppSchedulables().size()); @@ -661,8 +660,8 @@ public class TestFairScheduler { // submit app with empty queue ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); - AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( - appAttemptId, "", "user1"); + AppAttemptAddedSchedulerEvent appAddedEvent = + new AppAttemptAddedSchedulerEvent(appAttemptId, "", "user1"); scheduler.handle(appAddedEvent); // submission rejected @@ -787,11 +786,11 @@ public class TestFairScheduler { scheduler.reinitialize(conf, resourceManager.getRMContext()); ApplicationAttemptId id11 = createAppAttemptId(1, 1); - scheduler.addApplication(id11, "root.queue1", "user1"); + scheduler.addApplicationAttempt(id11, "root.queue1", "user1"); ApplicationAttemptId id21 = createAppAttemptId(2, 1); - scheduler.addApplication(id21, "root.queue2", "user1"); + scheduler.addApplicationAttempt(id21, "root.queue2", "user1"); ApplicationAttemptId id22 = createAppAttemptId(2, 2); - scheduler.addApplication(id22, "root.queue2", "user1"); + scheduler.addApplicationAttempt(id22, "root.queue2", "user1"); int minReqSize = FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB; @@ -833,8 +832,9 @@ public class TestFairScheduler { public void testAppAdditionAndRemoval() throws Exception { scheduler.reinitialize(conf, resourceManager.getRMContext()); - AppAddedSchedulerEvent appAddedEvent1 = new AppAddedSchedulerEvent( - createAppAttemptId(1, 1), "default", "user1"); + AppAttemptAddedSchedulerEvent appAddedEvent1 = + new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), "default", + "user1"); scheduler.handle(appAddedEvent1); // Scheduler should have two queues (the default and the one created for user1) @@ -844,7 +844,7 @@ public class TestFairScheduler { assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getRunnableAppSchedulables().size()); - AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent( createAppAttemptId(1, 1), RMAppAttemptState.FINISHED); // Now remove app @@ -1526,7 +1526,7 @@ public class TestFairScheduler { scheduler.handle(nodeEvent2); ApplicationAttemptId appId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - scheduler.addApplication(appId, "queue1", "user1"); + scheduler.addApplicationAttempt(appId, "queue1", "user1"); // 1 request with 2 nodes on the same rack. another request with 1 node on // a different rack @@ -1764,7 +1764,7 @@ public class TestFairScheduler { ApplicationAttemptId attId = ApplicationAttemptId.newInstance(applicationId, this.ATTEMPT_ID++); - scheduler.addApplication(attId, queue, user); + scheduler.addApplicationAttempt(attId, queue, user); numTries = 0; while (application.getFinishTime() == 0 && numTries < MAX_TRIES) { @@ -2238,8 +2238,8 @@ public class TestFairScheduler { verifyQueueNumRunnable("queue1", 2, 1); // Remove app 1 and both app 2 and app 4 should becomes runnable in its place - AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( - attId1, RMAppAttemptState.FINISHED); + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = + new AppAttemptRemovedSchedulerEvent(attId1, RMAppAttemptState.FINISHED); scheduler.handle(appRemovedEvent1); verifyAppRunnable(attId2, true); verifyQueueNumRunnable("queue2", 1, 0); @@ -2302,8 +2302,8 @@ public class TestFairScheduler { // Even though the app was removed from sub3, the app from sub2 gets to go // because it came in first - AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( - attId2, RMAppAttemptState.FINISHED); + AppAttemptRemovedSchedulerEvent appRemovedEvent1 = + new AppAttemptRemovedSchedulerEvent(attId2, RMAppAttemptState.FINISHED); scheduler.handle(appRemovedEvent1); verifyAppRunnable(attId4, true); verifyQueueNumRunnable("queue1.sub2", 2, 0); @@ -2311,16 +2311,16 @@ public class TestFairScheduler { verifyQueueNumRunnable("queue1.sub3", 0, 1); // Now test removal of a non-runnable app - AppRemovedSchedulerEvent appRemovedEvent2 = new AppRemovedSchedulerEvent( - attId5, RMAppAttemptState.KILLED); + AppAttemptRemovedSchedulerEvent appRemovedEvent2 = + new AppAttemptRemovedSchedulerEvent(attId5, RMAppAttemptState.KILLED); scheduler.handle(appRemovedEvent2); assertEquals(0, scheduler.maxRunningEnforcer.usersNonRunnableApps .get("user1").size()); // verify app gone in queue accounting verifyQueueNumRunnable("queue1.sub3", 0, 0); // verify it doesn't become runnable when there would be space for it - AppRemovedSchedulerEvent appRemovedEvent3 = new AppRemovedSchedulerEvent( - attId4, RMAppAttemptState.FINISHED); + AppAttemptRemovedSchedulerEvent appRemovedEvent3 = + new AppAttemptRemovedSchedulerEvent(attId4, RMAppAttemptState.FINISHED); scheduler.handle(appRemovedEvent3); verifyQueueNumRunnable("queue1.sub2", 1, 0); verifyQueueNumRunnable("queue1.sub3", 0, 0); @@ -2356,7 +2356,7 @@ public class TestFairScheduler { // send application request ApplicationAttemptId appAttemptId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); - fs.addApplication(appAttemptId, "queue11", "user11"); + fs.addApplicationAttempt(appAttemptId, "queue11", "user11"); List ask = new ArrayList(); ResourceRequest request = createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index e70070434ec..7ce7e42bc67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -64,7 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppRepor import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; @@ -150,13 +150,13 @@ public class TestFifoScheduler { ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); - SchedulerEvent event = new AppAddedSchedulerEvent(appAttemptId, "queue", - "user"); + SchedulerEvent event = + new AppAttemptAddedSchedulerEvent(appAttemptId, "queue", "user"); schedular.handle(event); appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2); - event = new AppAddedSchedulerEvent(appAttemptId, "queue", "user"); + event = new AppAttemptAddedSchedulerEvent(appAttemptId, "queue", "user"); schedular.handle(event); int afterAppsSubmitted = metrics.getAppsSubmitted(); @@ -188,8 +188,8 @@ public class TestFifoScheduler { int _appAttemptId = 1; ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId); - AppAddedSchedulerEvent appEvent1 = new AppAddedSchedulerEvent(appAttemptId, - "queue1", "user1"); + AppAttemptAddedSchedulerEvent appEvent1 = + new AppAttemptAddedSchedulerEvent(appAttemptId, "queue1", "user1"); scheduler.handle(appEvent1); int memory = 64; @@ -274,8 +274,8 @@ public class TestFifoScheduler { int _appAttemptId = 1; ApplicationAttemptId appAttemptId = createAppAttemptId(_appId, _appAttemptId); - AppAddedSchedulerEvent appEvent1 = new AppAddedSchedulerEvent(appAttemptId, - "queue1", "user1"); + AppAttemptAddedSchedulerEvent appEvent1 = + new AppAttemptAddedSchedulerEvent(appAttemptId, "queue1", "user1"); scheduler.handle(appEvent1); int memory = 1024; @@ -541,8 +541,8 @@ public class TestFifoScheduler { ApplicationId appId = BuilderUtils.newApplicationId(100, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); - SchedulerEvent event = new AppAddedSchedulerEvent(appAttemptId, "default", - "user"); + SchedulerEvent event = + new AppAttemptAddedSchedulerEvent(appAttemptId, "default", "user"); fs.handle(event); // Verify the blacklist can be updated independent of requesting containers From 8008929bf71442bc4d75069b5195c180468bb9d9 Mon Sep 17 00:00:00 2001 From: Jian He Date: Fri, 13 Dec 2013 01:59:30 +0000 Subject: [PATCH 75/76] Updated CHANGES.txt for YARN-1311. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550615 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ce954cf4cc4..15a2d8572c6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -165,6 +165,9 @@ Release 2.4.0 - UNRELEASED YARN-1325. Modified RM HA configuration validation to also ensure that multiple RMs are configured. (Xuan Gong via vinodkv) + YARN-1311. Fixed app specific scheduler-events' names to be app-attempt + based. (vinodkv via jianhe) + OPTIMIZATIONS BUG FIXES From a31a3a68ead2773f8daa5b3ac1041e271b442789 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Fri, 13 Dec 2013 02:47:04 +0000 Subject: [PATCH 76/76] HDFS-5652. Refactor invalid block token exception handling in DFSInputStream. (Liang Xie via junping_du) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1550620 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../apache/hadoop/hdfs/DFSInputStream.java | 50 +++++++++++-------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index de710e6ebef..b7715516bd5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -725,6 +725,9 @@ Release 2.4.0 - UNRELEASED HDFS-5637. Try to refeatchToken while local read InvalidToken occurred. (Liang Xie via junping_du) + HDFS-5652. Refactor invalid block token exception handling in DFSInputStream. + (Liang Xie via junping_du) + OPTIMIZATIONS HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index d24b0bc0757..bb9a066afe0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -591,21 +591,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, // The encryption key used is invalid. refetchEncryptionKey--; dfsClient.clearDataEncryptionKey(); - } else if ((ex instanceof InvalidBlockTokenException || ex instanceof InvalidToken) - && refetchToken > 0) { - DFSClient.LOG.info("Will fetch a new access token and retry, " - + "access token was invalid when connecting to " + targetAddr - + " : " + ex); - /* - * Get a new access token and retry. Retry is needed in 2 cases. 1) - * When both NN and DN re-started while DFSClient holding a cached - * access token. 2) In the case that NN fails to update its - * access key at pre-set interval (by a wide margin) and - * subsequently restarts. In this case, DN re-registers itself with - * NN and receives a new access key, but DN will delete the old - * access key from its memory since it's considered expired based on - * the estimated expiration date. - */ + } else if (refetchToken > 0 && tokenRefetchNeeded(ex, targetAddr)) { refetchToken--; fetchBlockAt(target); } else { @@ -978,11 +964,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, // The encryption key used is invalid. refetchEncryptionKey--; dfsClient.clearDataEncryptionKey(); - } else if ((e instanceof InvalidBlockTokenException || e instanceof InvalidToken) - && refetchToken > 0) { - DFSClient.LOG.info("Will get a new access token and retry, " - + "access token was invalid when connecting to " + targetAddr - + " : " + e); + } else if (refetchToken > 0 && tokenRefetchNeeded(e, targetAddr)) { refetchToken--; fetchBlockAt(block.getStartOffset()); continue; @@ -1003,6 +985,34 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead, } } + /** + * Should the block access token be refetched on an exception + * + * @param ex Exception received + * @param targetAddr Target datanode address from where exception was received + * @return true if block access token has expired or invalid and it should be + * refetched + */ + private static boolean tokenRefetchNeeded(IOException ex, + InetSocketAddress targetAddr) { + /* + * Get a new access token and retry. Retry is needed in 2 cases. 1) + * When both NN and DN re-started while DFSClient holding a cached + * access token. 2) In the case that NN fails to update its + * access key at pre-set interval (by a wide margin) and + * subsequently restarts. In this case, DN re-registers itself with + * NN and receives a new access key, but DN will delete the old + * access key from its memory since it's considered expired based on + * the estimated expiration date. + */ + if (ex instanceof InvalidBlockTokenException || ex instanceof InvalidToken) { + DFSClient.LOG.info("Access token was invalid when connecting to " + + targetAddr + " : " + ex); + return true; + } + return false; + } + private Peer newTcpPeer(InetSocketAddress addr) throws IOException { Peer peer = null; boolean success = false;